Merge pull request #305 from microlinkhq/next

feat: use in memory rate limiter

package.json

@@ -91,7 +91,6 @@
     "@microlink/mql": "~0.12.2",
     "@microlink/ping-url": "~1.4.11",
     "@microlink/ua": "~1.2.0",
-    "async-ratelimiter": "~1.3.13",
     "browserless": "~10.2.4",
     "cacheable-lookup": "~6.1.0",
     "cacheable-response": "~2.8.10",
@@ -119,6 +118,7 @@
     "p-timeout": "~4.1.0",
     "puppeteer": "~21.7.0",
     "qsm": "~2.1.2",
+    "rate-limiter-flexible": "~4.0.0",
     "router-http": "~1.0.5",
     "send-http": "~1.0.6",
     "serve-static": "~1.15.0",
@@ -182,7 +182,7 @@
     "release": "standard-version -a",
     "release:github": "github-generate-release",
     "release:tags": "git push --follow-tags origin HEAD:master",
-    "start": "TZ=UTC NODE_ENV=production DEBUG=\"unavatar*\" node src/server.js",
+    "start": "NODE_ENV=production node src/server.js",
     "test": "ava",
     "update": "ncu -u",
     "update:check": "ncu -- --error-level 2"

src/authentication.js

@@ -1,9 +1,14 @@
 'use strict'
 
-const rateLimiter = require('./util/rate-limiter')
+const { RateLimiterMemory } = require('rate-limiter-flexible')
 const debug = require('debug-logfmt')('unavatar:rate')
 
-const { API_KEY } = require('./constant')
+const { API_KEY, RATE_LIMIT_WINDOW, RATE_LIMIT } = require('./constant')
+
+const rateLimiter = new RateLimiterMemory({
+  points: RATE_LIMIT,
+  duration: RATE_LIMIT_WINDOW
+})
 
 const more = (() => {
   const email = 'hello@microlink.io'
@@ -30,21 +35,21 @@ const rateLimitError = (() => {
   return rateLimitError
 })()
 
-module.exports = rateLimiter
-  ? async (req, res, next) => {
-    if (req.headers['x-api-key'] === API_KEY) return next()
-    const { total, reset, remaining } = await rateLimiter.get({
-      id: req.ipAddress
-    })
+module.exports = async (req, res, next) => {
+  if (req.headers['x-api-key'] === API_KEY) return next()
 
-    if (!res.writableEnded) {
-      const _remaining = Math.max(0, remaining - 1)
-      res.setHeader('X-Rate-Limit-Limit', total)
-      res.setHeader('X-Rate-Limit-Remaining', _remaining)
-      res.setHeader('X-Rate-Limit-Reset', reset)
-      debug(req.ipAddress, { total, remaining: _remaining })
-    }
+  const { msBeforeNext, remainingPoints: remaining } = await rateLimiter
+    .consume(req.ipAddress)
+    .catch(error => error)
 
-    return remaining ? next() : next(rateLimitError)
+  if (!res.writableEnded) {
+    res.setHeader('X-Rate-Limit-Limit', RATE_LIMIT)
+    res.setHeader('X-Rate-Limit-Remaining', remaining)
+    res.setHeader('X-Rate-Limit-Reset', new Date(Date.now() + msBeforeNext))
+    debug(req.ipAddress, { total: RATE_LIMIT, remaining })
   }
-  : false
+
+  if (remaining) return next()
+  res.setHeader('Retry-After', msBeforeNext / 1000)
+  return next(rateLimitError)
+}

src/constant.js

@@ -11,7 +11,9 @@ const {
   AVATAR_TIMEOUT = 25000,
   CACHE_TTL = ms('1y'),
   NODE_ENV = 'development',
-  PORT = 3000
+  PORT = 3000,
+  RATE_LIMIT_WINDOW = 86400,
+  RATE_LIMIT = 50
 } = process.env
 
 const API_URL =
@@ -27,5 +29,7 @@ module.exports = {
   CACHE_TTL,
   NODE_ENV,
   PORT,
-  TMP_FOLDER
+  TMP_FOLDER,
+  RATE_LIMIT_WINDOW,
+  RATE_LIMIT
 }

src/index.js

@@ -40,6 +40,16 @@ router
       req.ipAddress = req.headers['cf-connecting-ip'] || '::ffff:127.0.0.1'
       next()
     },
+    require('helmet')({
+      crossOriginResourcePolicy: false,
+      contentSecurityPolicy: false
+    }),
+    require('http-compression')(),
+    require('cors')(),
+    serveStatic(path.resolve('public'), {
+      immutable: true,
+      maxAge: '1y'
+    }),
     require('./authentication'),
     isProduction && require('./ua'),
     (req, res, next) => {
@@ -63,17 +73,7 @@ router
         )
       })
       next()
-    },
-    require('helmet')({
-      crossOriginResourcePolicy: false,
-      contentSecurityPolicy: false
-    }),
-    require('http-compression')(),
-    require('cors')(),
-    serveStatic(path.resolve('public'), {
-      immutable: true,
-      maxAge: '1y'
-    })
+    }
   )
   .get('/:key', (req, res) =>
     ssrCache({

src/providers/twitter.js

@@ -4,7 +4,9 @@ const uniqueRandomArray = require('unique-random-array')
 const PCancelable = require('p-cancelable')
 const cheerio = require('cheerio')
 
-const randomCrawlerAgent = uniqueRandomArray(require('top-crawler-agents'))
+const randomCrawlerAgent = uniqueRandomArray(
+  require('top-crawler-agents').filter(agent => agent.startsWith('Slackbot'))
+)
 
 const getHTML = require('../util/html-get')
 
@@ -16,11 +18,7 @@ const avatarUrl = str =>
 
 module.exports = PCancelable.fn(async function twitter ({ input }, onCancel) {
   const promise = getHTML(`https://twitter.com/${input}`, {
-    headers: { 'user-agent': randomCrawlerAgent() },
-    puppeteerOpts: {
-      waitUntil: 'networkidle2',
-      abortTypes: ['image', 'stylesheet', 'font']
-    }
+    headers: { 'user-agent': randomCrawlerAgent() }
   })
   onCancel(() => promise.onCancel())
   const { html } = await promise

src/server.js

@@ -1,7 +1,8 @@
 'use strict'
 
 process.env.DEBUG =
-  process.env.DEBUG || '*,-puppeteer*,-send*,-ioredis*,-cacheable-response*'
+  process.env.DEBUG ||
+  '*,-html-get*,-puppeteer*,-send*,-ioredis*,-cacheable-response*,-browserless*'
 
 const debug = require('debug-logfmt')('unavatar')
 const { createServer } = require('http')

src/util/got.js

@@ -18,7 +18,6 @@ const userAgentHook = options => {
 const gotOpts = {
   dnsCache: require('./cacheable-lookup'),
   https: { rejectUnauthorized: false },
-  headers: { 'user-agent': undefined },
   hooks: { beforeRequest: [userAgentHook, tlsHook] }
 }