start reworking into a proper postgres setup

.pylintrc

@@ -0,0 +1 @@
+disable=C0111,C0103

docker-compose.yml

@@ -1,14 +1,29 @@
 scrounger:
+  container_name: scrounger
+  restart: always
   build: scrounger
   command: python -u app/scrounger.py
   ports:
     - "5000:5000"
   volumes:
     - ./scrounger:/app
   links:
-    - db
+    - postgres:postgres
   environment:
     - RUN_SCHEDULER=False
 
-db:
-  image: mongo:3.0.2
+data:
+  container_name: data
+  restart: always
+  image: postgres:latest
+  volumes:
+    - /var/lib/postgresql
+
+postgres:
+  container_name: postgres
+  restart: always
+  image: postgres:latest
+  volumes_from:
+    - data
+  ports:
+    - "5432:5432"

scrounger/Dockerfile

@@ -1,4 +1,5 @@
-FROM python:3.6.1-alpine
+FROM python:3.6.1
 ADD . /app
 WORKDIR /app
+ENV PYTHONPATH $PYTHONPATH:/app/app
 RUN pip install -r requirements.txt

scrounger/app/db/create_db.py

@@ -0,0 +1,4 @@
+from scrounger import db
+
+
+db.create_all()

scrounger/app/db/models.py

@@ -0,0 +1,18 @@
+from scrounger import db
+
+
+class PullRequest(db.Model):
+    __tablename__ = 'pull_requests'
+
+    id = db.Column(db.Integer, primary_key=True)
+    data = db.Column(db.JSON, nullable=False)
+
+    def __init__(self, data):
+        self.data = data
+
+
+class Team(db.Model):
+    __tablename__ = 'teams'
+
+    name = db.Column(db.Text, primary_key=True)
+    members = db.Column(db.JSON, nullable=False)

scrounger/app/scrounger.py

@@ -8,26 +8,25 @@
 import requests
 from bson.json_util import dumps
 from flask import Flask, request
+from flask_sqlalchemy import SQLAlchemy
 from flask_apscheduler import APScheduler
-from pymongo import MongoClient
 
 from graphql import query, flatten_response
-from settings import (
-    LOGGING, SCHEDULER_JOBS, SCHEDULER_TIMEZONE, SCHEDULER_API_ENABLED, ORGS_TO_TRACK, TOKEN,
-    RUN_SCHEDULER, TIMESTAMP_FORMAT
-)
+from settings import BaseConfig
 
+dictConfig(BaseConfig.LOGGING)
+logger = getLogger('main')
 
-dictConfig(LOGGING)
 app = Flask(__name__)
-client = MongoClient(os.environ['DB_PORT_27017_TCP_ADDR'], 27017)
-db = client.gitdb
-logger = getLogger('main')
+app.config.from_object(BaseConfig)
+db = SQLAlchemy(app)
+
+from db.models import *
 
 
 @app.route('/issues')
 def issues():
-    items = [item for item in db.gitdb.everything.find()]
+    items = [item for item in PullRequest.query.all()]
 
     return dumps(items), 200, {'Content-Type': 'application/json'}
 
@@ -76,10 +75,10 @@ def teams_issues(name):
 def update():
 
     url = 'https://api.github.com/graphql'
-    headers = {'Authorization': 'Bearer {}'.format(TOKEN)}
+    headers = {'Authorization': 'Bearer {}'.format(BaseConfig.TOKEN)}
 
     new_prs = {}
-    for org in ORGS_TO_TRACK:
+    for org in BaseConfig.ORGS_TO_TRACK:
         logger.debug('finding all issues in {}'.format(org))
 
         # using a % string interpolation here because our query has {} in it already
@@ -91,14 +90,18 @@ def update():
         new_prs.update(resp)
 
     logger.debug('dropping all issues and updating')
-    db.gitdb.everything.delete_many({})
+    deleted = PullRequest.query.delete()
+    logger.debug('deleted {} pull requests'.format(deleted))
 
     # filter out PRs that are greater than 90 days old
     for pr in new_prs.values():
         issue_time = pr['updated_at']
-        parsed_time = datetime(*(strptime(issue_time, TIMESTAMP_FORMAT))[:6])
+        parsed_time = datetime(*(strptime(issue_time, BaseConfig.TIMESTAMP_FORMAT))[:6])
         if datetime.now() - parsed_time < timedelta(days=90):
-            db.gitdb.everything.insert_one(pr)
+            new_pr = PullRequest(json.loads(pr))
+            db.session.add(new_pr)
+
+    db.session.commit()
 
     return 'success'
 
@@ -109,13 +112,7 @@ def healthcheck():
 
 
 if __name__ == "__main__":
-    if RUN_SCHEDULER:
-        app.config.update(
-            JOBS=SCHEDULER_JOBS,
-            SCHEDULER_API_ENABLED=SCHEDULER_API_ENABLED,
-            SCHEDULER_TIMEZONE=SCHEDULER_TIMEZONE
-        )
-
+    if BaseConfig.RUN_SCHEDULER:
         # this restricts the scheduler to running in the parent process
         if not app.debug or os.environ.get('WERKZEUG_RUN_MAIN') == 'true':
             logger.debug('starting scheduler')

scrounger/app/settings.py

@@ -1,49 +1,58 @@
 import os
 
-LOG_LEVEL = os.environ.get('LOG_LEVEL', 'DEBUG')
-
-LOGGING = {
-    'version': 1,
-    'disable_existing_loggers': True,
-    'formatters': {
-        'console': {
-            'format': '[%(asctime)s][%(levelname)s] %(name)s '
-                      '%(filename)s:%(funcName)s:%(lineno)d | %(message)s',
-        },
-    },
-    'handlers': {
-        'console': {
-            'level': 'DEBUG',
-            'class': 'logging.StreamHandler',
-            'formatter': 'console',
+class BaseConfig(object):
+    # logging stuff
+    LOG_LEVEL = os.environ.get('LOG_LEVEL', 'DEBUG')
+    LOGGING = {
+        'version': 1,
+        'disable_existing_loggers': True,
+        'formatters': {
+            'console': {
+                'format': '[%(asctime)s][%(levelname)s] %(name)s '
+                        '%(filename)s:%(funcName)s:%(lineno)d | %(message)s',
+            },
         },
-    },
-    'loggers': {
-        'main': {
-            'handlers': ['console'],
-            'level': 'DEBUG',
-            'propagate': False
+        'handlers': {
+            'console': {
+                'level': 'DEBUG',
+                'class': 'logging.StreamHandler',
+                'formatter': 'console',
+            },
         },
+        'loggers': {
+            'main': {
+                'handlers': ['console'],
+                'level': 'DEBUG',
+                'propagate': False
+            },
+        }
     }
-}
-
-TOKEN = os.environ.get('GITHUB_TOKEN', '66d9528f377307629fd1dbff70e1b64aeaa90dac')
-
-ORGS_TO_TRACK = os.environ.get('ORGS', 'HearstCorp,Hearst-Hatchery,HearstDigitalStudios').split(',')
-
-TIMESTAMP_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
-
-SCHEDULER_JOBS = [
-    {
-        'id': 'update',
-        'func': 'scrounger:update',
-        'trigger': 'interval',
-        'minutes': 15,
-    }
-]
-
-SCHEDULER_API_ENABLED = True
-
-SCHEDULER_TIMEZONE = "America/New_York"
 
-RUN_SCHEDULER = os.environ.get('RUN_SCHEDULER', 'False') == 'True'
+    # github-related settings
+    TOKEN = os.environ.get('GITHUB_TOKEN', '66d9528f377307629fd1dbff70e1b64aeaa90dac')
+    ORGS_TO_TRACK = os.environ.get('ORGS', 'HearstCorp,Hearst-Hatchery,HearstDigitalStudios').split(',')
+    TIMESTAMP_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
+
+    # scheduler settings
+    SCHEDULER_JOBS = [
+        {
+            'id': 'update',
+            'func': 'scrounger:update',
+            'trigger': 'interval',
+            'minutes': 15,
+        }
+    ]
+    SCHEDULER_API_ENABLED = True
+    SCHEDULER_TIMEZONE = "America/New_York"
+    RUN_SCHEDULER = os.environ.get('RUN_SCHEDULER', 'False') == 'True'
+
+    # postgres db settings
+    DB_NAME    = os.environ.get('DB_NAME', 'postgres')
+    DB_USER    = os.environ.get('DB_USER', 'admin')
+    DB_PASS    = os.environ.get('DB_PASS', 'admin')
+    DB_SERVICE = os.environ.get('DB_SERVICE', 'postgres')
+    DB_PORT    = os.environ.get('DB_PORT', '5432')
+    SQLALCHEMY_DATABASE_URI = 'postgresql://{0}:{1}@{2}:{3}/{4}'.format(
+        DB_USER, DB_PASS, DB_SERVICE, DB_PORT, DB_NAME
+    )
+    SQLALCHEMY_TRACK_MODIFICATIONS = False

scrounger/requirements.txt

@@ -1,4 +1,7 @@
 Flask
 Flask-APScheduler
+Flask-SQLAlchemy
+psycopg2
 pymongo
 requests
+SQLAlchemy