From 68dd60c4619b203837de79cf1d363dcaed2a47a8 Mon Sep 17 00:00:00 2001 From: grobolom Date: Sun, 11 Jun 2017 20:25:02 -0400 Subject: [PATCH] start reworking into a proper postgres setup --- .pylintrc | 1 + docker-compose.yml | 21 ++++++-- scrounger/Dockerfile | 3 +- scrounger/app/db/create_db.py | 4 ++ scrounger/app/db/models.py | 18 +++++++ scrounger/app/scrounger.py | 41 +++++++-------- scrounger/app/settings.py | 95 +++++++++++++++++++---------------- scrounger/requirements.txt | 3 ++ 8 files changed, 117 insertions(+), 69 deletions(-) create mode 100644 .pylintrc create mode 100644 scrounger/app/db/create_db.py create mode 100644 scrounger/app/db/models.py diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..d830cda --- /dev/null +++ b/.pylintrc @@ -0,0 +1 @@ +disable=C0111,C0103 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index bcd7f9d..d92ec30 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,6 @@ scrounger: + container_name: scrounger + restart: always build: scrounger command: python -u app/scrounger.py ports: @@ -6,9 +8,22 @@ scrounger: volumes: - ./scrounger:/app links: - - db + - postgres:postgres environment: - RUN_SCHEDULER=False -db: - image: mongo:3.0.2 +data: + container_name: data + restart: always + image: postgres:latest + volumes: + - /var/lib/postgresql + +postgres: + container_name: postgres + restart: always + image: postgres:latest + volumes_from: + - data + ports: + - "5432:5432" \ No newline at end of file diff --git a/scrounger/Dockerfile b/scrounger/Dockerfile index a406a74..3b205d5 100644 --- a/scrounger/Dockerfile +++ b/scrounger/Dockerfile @@ -1,4 +1,5 @@ -FROM python:3.6.1-alpine +FROM python:3.6.1 ADD . /app WORKDIR /app +ENV PYTHONPATH $PYTHONPATH:/app/app RUN pip install -r requirements.txt diff --git a/scrounger/app/db/create_db.py b/scrounger/app/db/create_db.py new file mode 100644 index 0000000..e19b1ff --- /dev/null +++ b/scrounger/app/db/create_db.py @@ -0,0 +1,4 @@ +from scrounger import db + + +db.create_all() \ No newline at end of file diff --git a/scrounger/app/db/models.py b/scrounger/app/db/models.py new file mode 100644 index 0000000..9081031 --- /dev/null +++ b/scrounger/app/db/models.py @@ -0,0 +1,18 @@ +from scrounger import db + + +class PullRequest(db.Model): + __tablename__ = 'pull_requests' + + id = db.Column(db.Integer, primary_key=True) + data = db.Column(db.JSON, nullable=False) + + def __init__(self, data): + self.data = data + + +class Team(db.Model): + __tablename__ = 'teams' + + name = db.Column(db.Text, primary_key=True) + members = db.Column(db.JSON, nullable=False) \ No newline at end of file diff --git a/scrounger/app/scrounger.py b/scrounger/app/scrounger.py index e986c03..54e076c 100644 --- a/scrounger/app/scrounger.py +++ b/scrounger/app/scrounger.py @@ -8,26 +8,25 @@ import requests from bson.json_util import dumps from flask import Flask, request +from flask_sqlalchemy import SQLAlchemy from flask_apscheduler import APScheduler -from pymongo import MongoClient from graphql import query, flatten_response -from settings import ( - LOGGING, SCHEDULER_JOBS, SCHEDULER_TIMEZONE, SCHEDULER_API_ENABLED, ORGS_TO_TRACK, TOKEN, - RUN_SCHEDULER, TIMESTAMP_FORMAT -) +from settings import BaseConfig +dictConfig(BaseConfig.LOGGING) +logger = getLogger('main') -dictConfig(LOGGING) app = Flask(__name__) -client = MongoClient(os.environ['DB_PORT_27017_TCP_ADDR'], 27017) -db = client.gitdb -logger = getLogger('main') +app.config.from_object(BaseConfig) +db = SQLAlchemy(app) + +from db.models import * @app.route('/issues') def issues(): - items = [item for item in db.gitdb.everything.find()] + items = [item for item in PullRequest.query.all()] return dumps(items), 200, {'Content-Type': 'application/json'} @@ -76,10 +75,10 @@ def teams_issues(name): def update(): url = 'https://api.github.com/graphql' - headers = {'Authorization': 'Bearer {}'.format(TOKEN)} + headers = {'Authorization': 'Bearer {}'.format(BaseConfig.TOKEN)} new_prs = {} - for org in ORGS_TO_TRACK: + for org in BaseConfig.ORGS_TO_TRACK: logger.debug('finding all issues in {}'.format(org)) # using a % string interpolation here because our query has {} in it already @@ -91,14 +90,18 @@ def update(): new_prs.update(resp) logger.debug('dropping all issues and updating') - db.gitdb.everything.delete_many({}) + deleted = PullRequest.query.delete() + logger.debug('deleted {} pull requests'.format(deleted)) # filter out PRs that are greater than 90 days old for pr in new_prs.values(): issue_time = pr['updated_at'] - parsed_time = datetime(*(strptime(issue_time, TIMESTAMP_FORMAT))[:6]) + parsed_time = datetime(*(strptime(issue_time, BaseConfig.TIMESTAMP_FORMAT))[:6]) if datetime.now() - parsed_time < timedelta(days=90): - db.gitdb.everything.insert_one(pr) + new_pr = PullRequest(json.loads(pr)) + db.session.add(new_pr) + + db.session.commit() return 'success' @@ -109,13 +112,7 @@ def healthcheck(): if __name__ == "__main__": - if RUN_SCHEDULER: - app.config.update( - JOBS=SCHEDULER_JOBS, - SCHEDULER_API_ENABLED=SCHEDULER_API_ENABLED, - SCHEDULER_TIMEZONE=SCHEDULER_TIMEZONE - ) - + if BaseConfig.RUN_SCHEDULER: # this restricts the scheduler to running in the parent process if not app.debug or os.environ.get('WERKZEUG_RUN_MAIN') == 'true': logger.debug('starting scheduler') diff --git a/scrounger/app/settings.py b/scrounger/app/settings.py index 022d381..7a6e533 100644 --- a/scrounger/app/settings.py +++ b/scrounger/app/settings.py @@ -1,49 +1,58 @@ import os -LOG_LEVEL = os.environ.get('LOG_LEVEL', 'DEBUG') - -LOGGING = { - 'version': 1, - 'disable_existing_loggers': True, - 'formatters': { - 'console': { - 'format': '[%(asctime)s][%(levelname)s] %(name)s ' - '%(filename)s:%(funcName)s:%(lineno)d | %(message)s', - }, - }, - 'handlers': { - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'formatter': 'console', +class BaseConfig(object): + # logging stuff + LOG_LEVEL = os.environ.get('LOG_LEVEL', 'DEBUG') + LOGGING = { + 'version': 1, + 'disable_existing_loggers': True, + 'formatters': { + 'console': { + 'format': '[%(asctime)s][%(levelname)s] %(name)s ' + '%(filename)s:%(funcName)s:%(lineno)d | %(message)s', + }, }, - }, - 'loggers': { - 'main': { - 'handlers': ['console'], - 'level': 'DEBUG', - 'propagate': False + 'handlers': { + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + 'formatter': 'console', + }, }, + 'loggers': { + 'main': { + 'handlers': ['console'], + 'level': 'DEBUG', + 'propagate': False + }, + } } -} - -TOKEN = os.environ.get('GITHUB_TOKEN', '66d9528f377307629fd1dbff70e1b64aeaa90dac') - -ORGS_TO_TRACK = os.environ.get('ORGS', 'HearstCorp,Hearst-Hatchery,HearstDigitalStudios').split(',') - -TIMESTAMP_FORMAT = '%Y-%m-%dT%H:%M:%SZ' - -SCHEDULER_JOBS = [ - { - 'id': 'update', - 'func': 'scrounger:update', - 'trigger': 'interval', - 'minutes': 15, - } -] - -SCHEDULER_API_ENABLED = True - -SCHEDULER_TIMEZONE = "America/New_York" -RUN_SCHEDULER = os.environ.get('RUN_SCHEDULER', 'False') == 'True' + # github-related settings + TOKEN = os.environ.get('GITHUB_TOKEN', '66d9528f377307629fd1dbff70e1b64aeaa90dac') + ORGS_TO_TRACK = os.environ.get('ORGS', 'HearstCorp,Hearst-Hatchery,HearstDigitalStudios').split(',') + TIMESTAMP_FORMAT = '%Y-%m-%dT%H:%M:%SZ' + + # scheduler settings + SCHEDULER_JOBS = [ + { + 'id': 'update', + 'func': 'scrounger:update', + 'trigger': 'interval', + 'minutes': 15, + } + ] + SCHEDULER_API_ENABLED = True + SCHEDULER_TIMEZONE = "America/New_York" + RUN_SCHEDULER = os.environ.get('RUN_SCHEDULER', 'False') == 'True' + + # postgres db settings + DB_NAME = os.environ.get('DB_NAME', 'postgres') + DB_USER = os.environ.get('DB_USER', 'admin') + DB_PASS = os.environ.get('DB_PASS', 'admin') + DB_SERVICE = os.environ.get('DB_SERVICE', 'postgres') + DB_PORT = os.environ.get('DB_PORT', '5432') + SQLALCHEMY_DATABASE_URI = 'postgresql://{0}:{1}@{2}:{3}/{4}'.format( + DB_USER, DB_PASS, DB_SERVICE, DB_PORT, DB_NAME + ) + SQLALCHEMY_TRACK_MODIFICATIONS = False \ No newline at end of file diff --git a/scrounger/requirements.txt b/scrounger/requirements.txt index 57a00b0..f9d1504 100644 --- a/scrounger/requirements.txt +++ b/scrounger/requirements.txt @@ -1,4 +1,7 @@ Flask Flask-APScheduler +Flask-SQLAlchemy +psycopg2 pymongo requests +SQLAlchemy