# Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.

#!/usr/bin/env python
#
import re, base64, logging, pickle, httplib2, time, urlparse, urllib2, urllib, StringIO, gzip, zipfile

from google.appengine.ext import webapp, db

from google.appengine.api import taskqueue, urlfetch, memcache, images, users
from google.appengine.ext.webapp.util import login_required
from google.appengine.ext.webapp import template

from django.utils import simplejson as json
from django.utils.html import strip_tags

from oauth2client.appengine import CredentialsProperty
from oauth2client.client import OAuth2WebServerFlow

import encoder

# TODO(jimhug): Allow client to request desired thumb size.
THUMB_SIZE = (57, 57)
READER_API = 'http://www.google.com/reader/api/0'

MAX_SECTIONS = 5
MAX_ARTICLES = 20

class UserData(db.Model):
  credentials = CredentialsProperty()
  sections = db.ListProperty(db.Key)

  def getEncodedData(self, articleKeys=None):
    enc = encoder.Encoder()
    # TODO(jimhug): Only return initially visible section in first reply.
    maxSections = min(MAX_SECTIONS, len(self.sections))
    enc.writeInt(maxSections)
    for section in db.get(self.sections[:maxSections]):
      section.encode(enc, articleKeys)
    return enc.getRaw()


class Section(db.Model):
  title = db.TextProperty()
  feeds = db.ListProperty(db.Key)

  def fixedTitle(self):
    return self.title.split('_')[0]

  def encode(self, enc, articleKeys=None):
    # TODO(jimhug): Need to optimize format and support incremental updates.
    enc.writeString(self.key().name())
    enc.writeString(self.fixedTitle())
    enc.writeInt(len(self.feeds))
    for feed in db.get(self.feeds):
      feed.ensureEncodedFeed()
      enc.writeRaw(feed.encodedFeed3)
      if articleKeys is not None:
        articleKeys.extend(feed.topArticles)

class Feed(db.Model):
  title = db.TextProperty()
  iconUrl = db.TextProperty()
  lastUpdated = db.IntegerProperty()

  encodedFeed3 = db.TextProperty()
  topArticles = db.ListProperty(db.Key)

  def ensureEncodedFeed(self, force=False):
    if force or self.encodedFeed3 is None:
      enc = encoder.Encoder()
      articleSet = []
      self.encode(enc, MAX_ARTICLES, articleSet)
      logging.info('articleSet length is %s' % len(articleSet))
      self.topArticles = articleSet
      self.encodedFeed3 = enc.getRaw()
      self.put()

  def encode(self, enc, maxArticles, articleSet):
    enc.writeString(self.key().name())
    enc.writeString(self.title)
    enc.writeString(self.iconUrl)

    logging.info('encoding feed: %s' % self.title)
    encodedArts = []

    for article in self.article_set.order('-date').fetch(limit=maxArticles):
      encodedArts.append(article.encodeHeader())
      articleSet.append(article.key())

    enc.writeInt(len(encodedArts))
    enc.writeRaw(''.join(encodedArts))


class Article(db.Model):
  feed = db.ReferenceProperty(Feed)

  title = db.TextProperty()
  author = db.TextProperty()
  content = db.TextProperty()
  snippet = db.TextProperty()
  thumbnail = db.BlobProperty()
  thumbnailSize = db.TextProperty()
  srcurl = db.TextProperty()
  date = db.IntegerProperty()

  def ensureThumbnail(self):
    # If our desired thumbnail size has changed, regenerate it and cache.
    if self.thumbnailSize != str(THUMB_SIZE):
       self.thumbnail = makeThumbnail(self.content)
       self.thumbnailSize = str(THUMB_SIZE)
       self.put()

  def encodeHeader(self):
    # TODO(jmesserly): for now always unescape until the crawler catches up
    enc = encoder.Encoder()
    enc.writeString(self.key().name())
    enc.writeString(unescape(self.title))
    enc.writeString(self.srcurl)
    enc.writeBool(self.thumbnail is not None)
    enc.writeString(self.author)
    enc.writeInt(self.date)
    enc.writeString(unescape(self.snippet))
    return enc.getRaw()

class HtmlFile(db.Model):
  content = db.BlobProperty()
  compressed = db.BooleanProperty()
  filename = db.StringProperty()
  author = db.UserProperty(auto_current_user=True)
  date = db.DateTimeProperty(auto_now_add=True)


class UpdateHtml(webapp.RequestHandler):
  def post(self):
    upload_files = self.request.POST.multi.__dict__['_items']
    version = self.request.get('version')
    logging.info('files: %r' % upload_files)
    for data in upload_files:
      if data[0] != 'files': continue
      file = data[1]
      filename = file.filename
      if version:
        filename = '%s-%s' % (version, filename)
      logging.info('upload: %r' % filename)

      htmlFile = HtmlFile.get_or_insert(filename)
      htmlFile.filename = filename

      # If text > (1MB - 1KB) then gzip text to fit in 1MB space
      text = file.value
      if len(text) > 1024*1023:
        data = StringIO.StringIO()
        gz = gzip.GzipFile(str(filename), 'wb', fileobj=data)
        gz.write(text)
        gz.close()
        htmlFile.content = data.getvalue()
        htmlFile.compressed = True
      else:
        htmlFile.content = text
        htmlFile.compressed = False

      htmlFile.put()

    self.redirect('/')

class TopHandler(webapp.RequestHandler):
  @login_required
  def get(self):
    user = users.get_current_user()
    prefs = UserData.get_by_key_name(user.user_id())
    if prefs is None:
      self.redirect('/update/user')
      return

    params = {'files': HtmlFile.all().order('-date').fetch(limit=30)}
    self.response.out.write(template.render('top.html', params))


class MainHandler(webapp.RequestHandler):

  @login_required
  def get(self, name):
    if name == 'dev':
     return self.handleDev()

    elif name == 'login':
      return self.handleLogin()

    elif name == 'upload':
      return self.handleUpload()

    user = users.get_current_user()
    prefs = UserData.get_by_key_name(user.user_id())
    if prefs is None:
      return self.handleLogin()

    html = HtmlFile.get_by_key_name(name)
    if html is None:
      self.error(404)
      return

    self.response.headers['Content-Type'] = 'text/html'

    if html.compressed:
      # TODO(jimhug): This slightly sucks ;-)
      # Can we write directly to the response.out?
      gz = gzip.GzipFile(name, 'rb', fileobj=StringIO.StringIO(html.content))
      self.response.out.write(gz.read())
      gz.close()
    else:
      self.response.out.write(html.content)

    # TODO(jimhug): Include first data packet with html.

  def handleLogin(self):
    user = users.get_current_user()
    # TODO(jimhug): Manage secrets for dart.googleplex.com better.
    # TODO(jimhug): Confirm that we need client_secret.
    flow = OAuth2WebServerFlow(
        client_id='267793340506.apps.googleusercontent.com',
        client_secret='5m8H-zyamfTYg5vnpYu1uGMU',
        scope=READER_API,
        user_agent='swarm')

    callback = self.request.relative_url('/oauth2callback')
    authorize_url = flow.step1_get_authorize_url(callback)

    memcache.set(user.user_id(), pickle.dumps(flow))

    content = template.render('login.html', {'authorize': authorize_url})
    self.response.out.write(content)

  def handleDev(self):
    user = users.get_current_user()
    content = template.render('dev.html', {'user': user})
    self.response.out.write(content)

  def handleUpload(self):
    user = users.get_current_user()
    content = template.render('upload.html', {'user': user})
    self.response.out.write(content)


class UploadFeed(webapp.RequestHandler):
  def post(self):
    upload_files = self.request.POST.multi.__dict__['_items']
    version = self.request.get('version')
    logging.info('files: %r' % upload_files)
    for data in upload_files:
      if data[0] != 'files': continue
      file = data[1]
      logging.info('upload feed: %r' % file.filename)

      data = json.loads(file.value)

      feedId = file.filename
      feed = Feed.get_or_insert(feedId)

      # Find the section to add it to.
      sectionTitle = data['section']
      section = findSectionByTitle(sectionTitle)
      if section != None:
        if feed.key() in section.feeds:
          logging.warn('Already contains feed %s, replacing' % feedId)
          section.feeds.remove(feed.key())

        # Add the feed to the section.
        section.feeds.insert(0, feed.key())
        section.put()

        # Add the articles.
        collectFeed(feed, data)

      else:
        logging.error('Could not find section %s to add the feed to' %
            sectionTitle)

    self.redirect('/')

# TODO(jimhug): Batch these up and request them more aggressively.
class DataHandler(webapp.RequestHandler):
  def get(self, name):
    if name.endswith('.jpg'):
      # Must be a thumbnail
      key = urllib2.unquote(name[:-len('.jpg')])
      article = Article.get_by_key_name(key)
      self.response.headers['Content-Type'] = 'image/jpeg'
      # cache images for 10 hours
      self.response.headers['Cache-Control'] = 'public,max-age=36000'
      article.ensureThumbnail()
      self.response.out.write(article.thumbnail)
    elif name.endswith('.html'):
      # Must be article content
      key = urllib2.unquote(name[:-len('.html')])
      article = Article.get_by_key_name(key)
      self.response.headers['Content-Type'] = 'text/html'
      if article is None:
        content = '<h2>Missing article</h2>'
      else:
        content = article.content
      # cache article content for 10 hours
      self.response.headers['Cache-Control'] = 'public,max-age=36000'
      self.response.out.write(content)
    elif name == 'user.data':
      self.response.out.write(self.getUserData())
    elif name == 'CannedData.dart':
      self.canData()
    elif name == 'CannedData.zip':
      self.canDataZip()
    else:
      self.error(404)

  def getUserData(self, articleKeys=None):
    user = users.get_current_user()
    user_id = user.user_id()

    key = 'data_' + user_id
    # need to flush memcache fairly frequently...
    data = memcache.get(key)
    if data is None:
      prefs = UserData.get_or_insert(user_id)
      if prefs is None:
        # TODO(jimhug): Graceful failure for unknown users.
        pass
      data = prefs.getEncodedData(articleKeys)
      # TODO(jimhug): memcache.set(key, data)

    return data

  def canData(self):
    def makeDartSafe(data):
      return repr(unicode(data))[1:].replace('$', '\\$')

    lines = ['// TODO(jimhug): Work out correct copyright for this file.',
             'class CannedData {']

    user = users.get_current_user()
    prefs = UserData.get_by_key_name(user.user_id())
    articleKeys = []
    data = prefs.getEncodedData(articleKeys)
    lines.append('  static const Map<String,String> data = const {')
    for article in db.get(articleKeys):
      key = makeDartSafe(urllib.quote(article.key().name())+'.html')
      lines.append('    %s:%s, ' % (key, makeDartSafe(article.content)))

    lines.append('    "user.data":%s' % makeDartSafe(data))

    lines.append('  };')

    lines.append('}')
    self.response.headers['Content-Type'] = 'application/dart'
    self.response.out.write('\n'.join(lines))

  # Get canned static data
  def canDataZip(self):
    # We need to zip into an in-memory buffer to get the right string encoding
    # behavior.
    data = StringIO.StringIO()
    result = zipfile.ZipFile(data, 'w')

    articleKeys = []
    result.writestr('data/user.data',
        self.getUserData(articleKeys).encode('utf-8'))
    logging.info('  adding articles %s' % len(articleKeys))
    images = []
    for article in db.get(articleKeys):
      article.ensureThumbnail()
      path = 'data/' + article.key().name() + '.html'
      result.writestr(path.encode('utf-8'), article.content.encode('utf-8'))
      if article.thumbnail:
        path = 'data/' + article.key().name() + '.jpg'
        result.writestr(path.encode('utf-8'), article.thumbnail)

    result.close()
    logging.info('writing CannedData.zip')
    self.response.headers['Content-Type'] = 'multipart/x-zip'
    disposition = 'attachment; filename=CannedData.zip'
    self.response.headers['Content-Disposition'] = disposition
    self.response.out.write(data.getvalue())
    data.close()


class SetDefaultFeeds(webapp.RequestHandler):
  @login_required
  def get(self):
    user = users.get_current_user()
    prefs = UserData.get_or_insert(user.user_id())

    prefs.sections = [
      db.Key.from_path('Section', 'user/17857667084667353155/label/Top'),
      db.Key.from_path('Section', 'user/17857667084667353155/label/Design'),
      db.Key.from_path('Section', 'user/17857667084667353155/label/Eco'),
      db.Key.from_path('Section', 'user/17857667084667353155/label/Geek'),
      db.Key.from_path('Section', 'user/17857667084667353155/label/Google'),
      db.Key.from_path('Section', 'user/17857667084667353155/label/Seattle'),
      db.Key.from_path('Section', 'user/17857667084667353155/label/Tech'),
      db.Key.from_path('Section', 'user/17857667084667353155/label/Web')]

    prefs.put()

    self.redirect('/')

class SetTestFeeds(webapp.RequestHandler):
  @login_required
  def get(self):
    user = users.get_current_user()
    prefs = UserData.get_or_insert(user.user_id())

    sections = []
    for i in range(3):
      s1 = Section.get_or_insert('Test%d' % i)
      s1.title = 'Section %d' % (i+1)

      feeds = []
      for j in range(4):
        label = '%d_%d' % (i, j)
        f1 = Feed.get_or_insert('Test%s' % label)
        f1.title = 'Feed %s' % label
        f1.iconUrl = getFeedIcon('http://google.com')
        f1.lastUpdated = 0
        f1.put()
        feeds.append(f1.key())

        for k in range(8):
          label = '%d_%d_%d' % (i, j, k)
          a1 = Article.get_or_insert('Test%s' % label)
          if a1.title is None:
            a1.feed = f1
            a1.title = 'Article %s' % label
            a1.author = 'anon'
            a1.content = 'Lorem ipsum something or other...'
            a1.snippet = 'Lorem ipsum something or other...'
            a1.thumbnail = None
            a1.srcurl = ''
            a1.date = 0

      s1.feeds = feeds
      s1.put()
      sections.append(s1.key())

    prefs.sections = sections
    prefs.put()

    self.redirect('/')


class UserLoginHandler(webapp.RequestHandler):
  @login_required
  def get(self):
    user = users.get_current_user()
    prefs = UserData.get_or_insert(user.user_id())
    if prefs.credentials:
      http = prefs.credentials.authorize(httplib2.Http())

      response, content = http.request('%s/subscription/list?output=json' %
                                       READER_API)
      self.collectFeeds(prefs, content)
      self.redirect('/')
    else:
      self.redirect('/login')


  def collectFeeds(self, prefs, content):
    data = json.loads(content)

    queue_name = self.request.get('queue_name', 'priority-queue')
    sections = {}
    for feedData in data['subscriptions']:
      feed = Feed.get_or_insert(feedData['id'])
      feed.put()
      category = feedData['categories'][0]
      categoryId = category['id']
      if not sections.has_key(categoryId):
        sections[categoryId] = (category['label'], [])

      # TODO(jimhug): Use Reader preferences to sort feeds in a section.
      sections[categoryId][1].append(feed.key())

      # Kick off a high priority feed update
      taskqueue.add(url='/update/feed', queue_name=queue_name,
                    params={'id': feed.key().name()})

    sectionKeys = []
    for name, (title, feeds) in sections.items():
      section = Section.get_or_insert(name)
      section.feeds = feeds
      section.title = title
      section.put()
      # Forces Top to be the first section
      if title == 'Top': title = '0Top'
      sectionKeys.append( (title, section.key()) )

    # TODO(jimhug): Use Reader preferences API to get users true sort order.
    prefs.sections = [key for t, key in sorted(sectionKeys)]
    prefs.put()


class AllFeedsCollector(webapp.RequestHandler):
  '''Ensures that a given feed object is locally up to date.'''
  def post(self): return self.get()

  def get(self):
    queue_name = self.request.get('queue_name', 'background')
    for feed in Feed.all():
      taskqueue.add(url='/update/feed', queue_name=queue_name,
                    params={'id': feed.key().name()})

UPDATE_COUNT = 4 # The number of articles to request on periodic updates.
INITIAL_COUNT = 40 # The number of articles to get first for a new queue.
SNIPPET_SIZE = 180 # The length of plain-text snippet to extract.
class FeedCollector(webapp.RequestHandler):
  def post(self): return self.get()

  def get(self):
    feedId = self.request.get('id')
    feed = Feed.get_or_insert(feedId)

    if feed.lastUpdated is None:
      self.fetchn(feed, feedId, INITIAL_COUNT)
    else:
      self.fetchn(feed, feedId, UPDATE_COUNT)

    self.response.headers['Content-Type'] = "text/plain"

  def fetchn(self, feed, feedId, n, continuation=None):
    # basic pattern is to read by ARTICLE_COUNT until we hit existing.
    if continuation is None:
      apiUrl = '%s/stream/contents/%s?n=%d' % (
        READER_API, feedId, n)
    else:
      apiUrl = '%s/stream/contents/%s?n=%d&c=%s' % (
        READER_API, feedId, n, continuation)

    logging.info('fetching: %s' % apiUrl)
    result = urlfetch.fetch(apiUrl)

    if result.status_code == 200:
      data = json.loads(result.content)
      collectFeed(feed, data, continuation)
    elif result.status_code == 401:
      self.response.out.write( '<pre>%s</pre>' % result.content)
    else:
      self.response.out.write(result.status_code)

def findSectionByTitle(title):
  for section in Section.all():
    if section.fixedTitle() == title:
      return section
  return None

def collectFeed(feed, data, continuation=None):
  '''
  Reads a feed from the given JSON object and populates the given feed object
  in the datastore with its data.
  '''
  if continuation is None:
    if 'alternate' in data:
      feed.iconUrl = getFeedIcon(data['alternate'][0]['href'])
    feed.title = data['title']
    feed.lastUpdated = data['updated']

  articles = data['items']
  logging.info('%d new articles for %s' % (len(articles), feed.title))

  for articleData in articles:
    if not collectArticle(feed, articleData):
      feed.put()
      return False

  if len(articles) > 0 and data.has_key('continuation'):
    logging.info('would have looked for more articles')
    # TODO(jimhug): Enable this continuation check when more robust
    #self.fetchn(feed, feedId, data['continuation'])

  feed.ensureEncodedFeed(force=True)
  feed.put()
  return True

def collectArticle(feed, data):
  '''
  Reads an article from the given JSON object and populates the datastore with
  it.
  '''
  if not 'title' in data:
    # Skip this articles without titles
    return True

  articleId = data['id']
  article = Article.get_or_insert(articleId)
  # TODO(jimhug): This aborts too early - at lease for one adafruit case.
  if article.date == data['published']:
    logging.info('found existing, aborting: %r, %r' %
      (articleId, article.date))
    return False

  if data.has_key('content'):
    content = data['content']['content']
  elif data.has_key('summary'):
    content = data['summary']['content']
  else:
    content = ''
  #TODO(jimhug): better summary?
  article.content = content
  article.date = data['published']
  article.title = unescape(data['title'])
  article.snippet = unescape(strip_tags(content)[:SNIPPET_SIZE])

  article.feed = feed

  # TODO(jimhug): make this canonical so UX can change for this state
  article.author = data.get('author', 'anonymous')

  article.ensureThumbnail()

  article.srcurl = ''
  if data.has_key('alternate'):
    for alt in data['alternate']:
      if alt.has_key('href'):
        article.srcurl = alt['href']
  return True

def unescape(html):
  "Inverse of Django's utils.html.escape function"
  if not isinstance(html, basestring):
      html = str(html)
  html = html.replace('&#39;', "'").replace('&quot;', '"')
  return html.replace('&gt;', '>').replace('&lt;', '<').replace('&amp;', '&')

def getFeedIcon(url):
  url = urlparse.urlparse(url).netloc
  return 'http://s2.googleusercontent.com/s2/favicons?domain=%s&alt=feed' % url

def findImage(text):
  img = findImgTag(text, 'jpg|jpeg|png')
  if img is not None:
    return img

  img = findVideoTag(text)
  if img is not None:
    return img

  img = findImgTag(text, 'gif')
  return img

def findImgTag(text, extensions):
  m = re.search(r'src="(http://\S+\.(%s))(\?.*)?"' % extensions, text)
  if m is None:
    return None
  return m.group(1)

def findVideoTag(text):
  # TODO(jimhug): Add other videos beyond youtube.
  m = re.search(r'src="http://www.youtube.com/(\S+)/(\S+)[/|"]', text)
  if m is None:
    return None

  return 'http://img.youtube.com/vi/%s/0.jpg' % m.group(2)

def makeThumbnail(text):
  url = None
  try:
    url = findImage(text)
    if url is None:
      return None
    return generateThumbnail(url)
  except:
    logging.info('error decoding: %s' % (url or text))
    return None

def generateThumbnail(url):
  logging.info('generating thumbnail: %s' % url)
  thumbWidth, thumbHeight = THUMB_SIZE

  result = urlfetch.fetch(url)
  img = images.Image(result.content)

  w, h = img.width, img.height

  aspect = float(w) / h
  thumbAspect = float(thumbWidth) / thumbHeight

  if aspect > thumbAspect:
    # Too wide, so crop on the sides.
    normalizedCrop = (w - h * thumbAspect) / (2.0 * w)
    img.crop(normalizedCrop, 0., 1. - normalizedCrop, 1. )
  elif aspect < thumbAspect:
    # Too tall, so crop out the bottom.
    normalizedCrop = (h - w / thumbAspect) / h
    img.crop(0., 0., 1., 1. - normalizedCrop)

  img.resize(thumbWidth, thumbHeight)

  # Chose JPEG encoding because informal experiments showed it generated
  # the best size to quality ratio for thumbnail images.
  nimg = img.execute_transforms(output_encoding=images.JPEG)
  logging.info('  finished thumbnail: %s' % url)

  return nimg

class OAuthHandler(webapp.RequestHandler):

  @login_required
  def get(self):
    user = users.get_current_user()
    flow = pickle.loads(memcache.get(user.user_id()))
    if flow:
      prefs = UserData.get_or_insert(user.user_id())
      prefs.credentials = flow.step2_exchange(self.request.params)
      prefs.put()
      self.redirect('/update/user')
    else:
      pass


def main():
  application = webapp.WSGIApplication(
      [
      ('/data/(.*)', DataHandler),

      # This is called periodically from cron.yaml.
      ('/update/allFeeds', AllFeedsCollector),
      ('/update/feed', FeedCollector),
      ('/update/user', UserLoginHandler),
      ('/update/defaultFeeds', SetDefaultFeeds),
      ('/update/testFeeds', SetTestFeeds),
      ('/update/html', UpdateHtml),
      ('/update/upload', UploadFeed),
      ('/oauth2callback', OAuthHandler),

      ('/', TopHandler),
      ('/(.*)', MainHandler),
      ],
      debug=True)
  webapp.util.run_wsgi_app(application)

if __name__ == '__main__':
  main()