Hydro exit event (#16067)

* Start a basic exit event

* Combine to one performance function

* Lint

* Fancy operators

* Update events.js

* Update events.js

* Update cookie-settings.js

* Add scroll tracking

* Tell "standard" to use babel-eslint

* Throttle scroll tracking

* Lint

* Use sendBeacon

* Update index.js

javascripts/events.js

@@ -6,7 +6,13 @@ import parseUserAgent from './user-agent'
 
 const COOKIE_NAME = '_docs-events'
 
+const startVisitTime = Date.now()
+
 let cookieValue
+let pageEventId
+let maxScrollY = 0
+let pauseScrolling = false
+let sentExit = false
 
 export function getUserEventsId () {
   if (cookieValue) return cookieValue
@@ -42,76 +48,130 @@ export async function sendEvent ({
   experiment_variation,
   experiment_success
 }) {
-  const response = await fetch('/events', {
-    method: 'POST',
-    headers: {
-      'Content-Type': 'application/json',
-      'CSRF-Token': getCsrf()
+  const body = {
+    _csrf: getCsrf(),
+
+    type, // One of page, exit, link, search, navigate, survey, experiment
+
+    context: {
+      // Primitives
+      event_id: uuidv4(),
+      user: getUserEventsId(),
+      version,
+      created: new Date().toISOString(),
+
+      // Content information
+      path: location.pathname,
+      referrer: document.referrer,
+      search: location.search,
+      href: location.href,
+      site_language: location.pathname.split('/')[1],
+
+      // Device information
+      // os, os_version, browser, browser_version:
+      ...parseUserAgent(),
+      viewport_width: document.documentElement.clientWidth,
+      viewport_height: document.documentElement.clientHeight,
+
+      // Location information
+      timezone: new Date().getTimezoneOffset() / -60,
+      user_language: navigator.language
     },
-    body: JSON.stringify({
-      type, // One of page, exit, link, search, navigate, survey, experiment
-
-      context: {
-        // Primitives
-        event_id: uuidv4(),
-        user: getUserEventsId(),
-        version,
-        created: new Date().toISOString(),
-
-        // Content information
-        path: location.pathname,
-        referrer: document.referrer,
-        search: location.search,
-        href: location.href,
-        site_language: location.pathname.split('/')[1],
-
-        // Device information
-        // os, os_version, browser, browser_version:
-        ...parseUserAgent(),
-        viewport_width: document.documentElement.clientWidth,
-        viewport_height: document.documentElement.clientHeight,
-
-        // Location information
-        timezone: new Date().getTimezoneOffset() / -60,
-        user_language: navigator.language
-      },
-
-      // Page event
-      page_render_duration,
-
-      // Exit event
-      exit_page_id,
-      exit_first_paint,
-      exit_dom_interactive,
-      exit_dom_complete,
-      exit_visit_duration,
-      exit_scroll_length,
-
-      // Link event
-      link_url,
-
-      // Search event
-      search_query,
-      search_context,
-
-      // Navigate event
-      navigate_label,
-
-      // Survey event
-      survey_vote,
-      survey_comment,
-      survey_email,
-
-      // Experiment event
-      experiment_name,
-      experiment_variation,
-      experiment_success
-    })
+
+    // Page event
+    page_render_duration,
+
+    // Exit event
+    exit_page_id,
+    exit_first_paint,
+    exit_dom_interactive,
+    exit_dom_complete,
+    exit_visit_duration,
+    exit_scroll_length,
+
+    // Link event
+    link_url,
+
+    // Search event
+    search_query,
+    search_context,
+
+    // Navigate event
+    navigate_label,
+
+    // Survey event
+    survey_vote,
+    survey_comment,
+    survey_email,
+
+    // Experiment event
+    experiment_name,
+    experiment_variation,
+    experiment_success
+  }
+  const blob = new Blob([JSON.stringify(body)], { type: 'application/json' })
+  navigator.sendBeacon('/events', blob)
+  return body
+}
+
+function getPerformance () {
+  const paint = performance?.getEntriesByType('paint')?.find(
+    ({ name }) => name === 'first-contentful-paint'
+  )
+  const nav = performance?.getEntriesByType('navigation')?.[0]
+  return {
+    firstContentfulPaint: paint ? paint / 1000 : undefined,
+    domInteractive: nav ? nav.domInteractive / 1000 : undefined,
+    domComplete: nav ? nav.domComplete / 1000 : undefined,
+    render: nav ? (nav.responseEnd - nav.requestStart) / 1000 : undefined
+  }
+}
+
+function trackScroll () {
+  // Throttle the calculations to no more than five per second
+  if (pauseScrolling) return
+  pauseScrolling = true
+  setTimeout(() => { pauseScrolling = false }, 200)
+
+  // Update maximum scroll position reached
+  const scrollPosition = (
+    (window.scrollY + window.innerHeight) /
+    document.documentElement.scrollHeight
+  )
+  if (scrollPosition > maxScrollY) maxScrollY = scrollPosition
+}
+
+async function sendExit () {
+  if (sentExit) return
+  if (document.visibilityState !== 'hidden') return
+  if (!pageEventId) return
+  sentExit = true
+  const {
+    firstContentfulPaint,
+    domInteractive,
+    domComplete
+  } = getPerformance()
+  return sendEvent({
+    type: 'exit',
+    exit_page_id: pageEventId,
+    exit_first_paint: firstContentfulPaint,
+    exit_dom_interactive: domInteractive,
+    exit_dom_complete: domComplete,
+    exit_visit_duration: (Date.now() - startVisitTime) / 1000,
+    exit_scroll_length: maxScrollY
   })
-  const data = response.ok ? await response.json() : {}
-  return data
 }
 
 export default async function initializeEvents () {
-  await sendEvent({ type: 'page' })
+  // Page event
+  const { render } = getPerformance()
+  const pageEvent = await sendEvent({
+    type: 'page',
+    page_render_duration: render
+  })
+
+  // Exit event
+  pageEventId = pageEvent?.context?.event_id
+  window.addEventListener('scroll', trackScroll)
+  document.addEventListener('visibilitychange', sendExit)
 }

lib/cookie-settings.js

@@ -1,7 +1,9 @@
 module.exports = {
   httpOnly: true, // can't access these cookies through browser JavaScript
-  secure: process.env.NODE_ENV !== 'test', // requires https protocol
+  secure: !['test', 'development'].includes(process.env.NODE_ENV),
+  // requires https protocol
   // `secure` doesn't work with supertest at all
+  // http://localhost fails on chrome with secure
   sameSite: 'lax'
   // most browsers are "lax" these days,
   // but older browsers used to default to "none"

middleware/events.js

@@ -10,13 +10,16 @@ const ajv = new Ajv()
 const router = express.Router()
 
 router.post('/', async (req, res, next) => {
-  if (!ajv.validate(schema, req.body)) {
+  const fields = omit(req.body, '_csrf')
+  if (!ajv.validate(schema, fields)) {
     if (process.env.NODE_ENV === 'development') console.log(ajv.errorsText())
     return res.status(400).json({})
   }
-  const fields = omit(req.body, OMIT_FIELDS)
   try {
-    const hydroRes = await req.hydro.publish(req.hydro.schemas[req.body.type], fields)
+    const hydroRes = await req.hydro.publish(
+      req.hydro.schemas[fields.type],
+      omit(fields, OMIT_FIELDS)
+    )
     if (!hydroRes.ok) return res.status(502).json({})
     return res.status(201).json(fields)
   } catch (err) {

middleware/index.js

@@ -27,12 +27,12 @@ module.exports = function (app) {
   app.use(require('./req-utils'))
   app.use(require('./robots'))
   app.use(require('./cookie-parser'))
+  app.use(express.json()) // Must come before ./csrf
   app.use(require('./csrf'))
   app.use(require('./handle-csrf-errors'))
   app.use(require('compression')())
   app.use(require('connect-slashes')(false))
   app.use('/dist', express.static('dist'))
-  app.use(express.json())
   app.use('/events', require('./events'))
   app.use(require('./categories-for-support-team'))
   app.use(require('./enterprise-data-endpoint'))

package.json

@@ -78,13 +78,15 @@
     "async": "^3.2.0",
     "await-sleep": "0.0.1",
     "aws-sdk": "^2.610.0",
+    "babel-eslint": "^10.1.0",
     "broken-link-checker": "^0.7.8",
     "chalk": "^4.0.0",
     "commander": "^2.20.3",
     "count-array-values": "^1.2.1",
     "csp-parse": "0.0.2",
     "csv-parse": "^4.8.8",
     "csv-parser": "^2.3.3",
+    "dedent": "^0.7.0",
     "del": "^4.1.1",
     "dependency-check": "^4.1.0",
     "domwaiter": "^1.1.0",
@@ -103,6 +105,7 @@
     "make-promises-safe": "^5.1.0",
     "mime": "^2.4.4",
     "mock-express-response": "^0.2.2",
+    "nock": "^13.0.4",
     "nodemon": "^2.0.4",
     "npm-merge-driver-install": "^1.1.1",
     "object-hash": "^2.0.1",
@@ -115,9 +118,7 @@
     "start-server-and-test": "^1.11.3",
     "supertest": "^4.0.2",
     "webpack-dev-middleware": "^3.7.2",
-    "website-scraper": "^4.2.0",
-    "dedent": "^0.7.0",
-    "nock": "^13.0.4"
+    "website-scraper": "^4.2.0"
   },
   "scripts": {
     "start": "cross-env NODE_ENV=development ENABLED_LANGUAGES='en,ja' nodemon server.js",
@@ -145,6 +146,7 @@
   },
   "repository": "https://github.com/github/docs",
   "standard": {
+    "parser": "babel-eslint",
     "env": [
       "browser",
       "jest"