diff --git a/README.md b/README.md index 7f21262..4bf5c83 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,14 @@ This package includes functions to process the telemetry. +# Download Data + +The `DownloadData` function will fetch data about the number of +downloads from the Chrome and Firefox extension stores. + # Retention Data The [`RetentionData`][] will calculate how many users have been using the extension for one day or two days in the last 28 days. -[`RetentionData`]: https://github.com/webhintio/hint/issues/3074 \ No newline at end of file +[`RetentionData`]: https://github.com/webhintio/hint/issues/3074 diff --git a/function-install-data/function.json b/function-install-data/function.json new file mode 100644 index 0000000..3de905e --- /dev/null +++ b/function-install-data/function.json @@ -0,0 +1,11 @@ +{ + "bindings": [ + { + "name": "processDailyInstalls", + "type": "timerTrigger", + "direction": "in", + "schedule": "0 0 11 * * *" + } + ], + "scriptFile": "../dist/src/install-data/index.js" +} diff --git a/package-lock.json b/package-lock.json index 8b89388..97beba7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -491,6 +491,15 @@ "integrity": "sha512-SQaNzWQ2YZSr7FqAyPPiA3FYpux2Lqh3HWMZQk47x3xbMCqgC/w0dY3dw9rGqlweDDkrySQBcaScXWeR+Yb11Q==", "dev": true }, + "@types/puppeteer": { + "version": "1.20.2", + "resolved": "https://registry.npmjs.org/@types/puppeteer/-/puppeteer-1.20.2.tgz", + "integrity": "sha512-oSFCtftHSfVx8K9XPdNNYs79Zt4pYJs/0NP78ltuGCB25zS3UNGJSiypBfbhbvRC5Dcsh0k1R5Z0i8HHtqQUPQ==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/sinon": { "version": "7.5.0", "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-7.5.0.tgz", @@ -604,6 +613,14 @@ "integrity": "sha512-tMUqwBWfLFbJbizRmEcWSLw6HnFzfdJs2sOJEOwwtVPMoH/0Ay+E703oZz78VSXZiiDcZrQ5XKjPIUQixhmgVw==", "dev": true }, + "agent-base": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.3.0.tgz", + "integrity": "sha512-salcGninV0nPrwpGNn4VTXBb1SOuXQBiqbrNXoeizJsHrsL6ERFM2Ne3JUSBWRE6aeNJI2ROP/WEEIDUiDe3cg==", + "requires": { + "es6-promisify": "^5.0.0" + } + }, "ajv": { "version": "6.10.2", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.2.tgz", @@ -790,6 +807,11 @@ "stack-chain": "^1.3.7" } }, + "async-limiter": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz", + "integrity": "sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==" + }, "async-listener": { "version": "0.6.10", "resolved": "https://registry.npmjs.org/async-listener/-/async-listener-0.6.10.tgz", @@ -919,8 +941,7 @@ "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", - "dev": true + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, "binary-extensions": { "version": "2.0.0", @@ -991,7 +1012,6 @@ "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -1009,8 +1029,7 @@ "buffer-from": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", - "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", - "dev": true + "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" }, "cacheable-request": { "version": "6.1.0", @@ -1403,8 +1422,47 @@ "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", - "dev": true + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + }, + "concat-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", + "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "requires": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^2.2.2", + "typedarray": "^0.0.6" + }, + "dependencies": { + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } }, "concordance": { "version": "4.0.0", @@ -1546,8 +1604,7 @@ "core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", - "dev": true + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, "cp-file": { "version": "6.2.0", @@ -1613,7 +1670,6 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", - "dev": true, "requires": { "ms": "^2.1.1" } @@ -1906,6 +1962,19 @@ "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", "dev": true }, + "es6-promise": { + "version": "4.2.8", + "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz", + "integrity": "sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w==" + }, + "es6-promisify": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", + "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=", + "requires": { + "es6-promise": "^4.0.3" + } + }, "escape-string-regexp": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", @@ -2433,6 +2502,32 @@ "tmp": "^0.0.33" } }, + "extract-zip": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-1.6.7.tgz", + "integrity": "sha1-qEC0uK9kAyZMjbV/Txp0Mz74H+k=", + "requires": { + "concat-stream": "1.6.2", + "debug": "2.6.9", + "mkdirp": "0.5.1", + "yauzl": "2.4.1" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + } + } + }, "fast-deep-equal": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", @@ -2479,6 +2574,14 @@ "reusify": "^1.0.0" } }, + "fd-slicer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.0.1.tgz", + "integrity": "sha1-i1vL2ewyfFBBv5qwI/1nUPEXfmU=", + "requires": { + "pend": "~1.2.0" + } + }, "figures": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/figures/-/figures-3.0.0.tgz", @@ -2664,8 +2767,7 @@ "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", - "dev": true + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" }, "fsevents": { "version": "2.1.1", @@ -2721,7 +2823,6 @@ "version": "7.1.4", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz", "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==", - "dev": true, "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -2871,6 +2972,25 @@ "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.0.3.tgz", "integrity": "sha512-TcIMG3qeVLgDr1TEd2XvHaTnMPwYQUQMIBLy+5pLSDKYFc7UIqj39w8EGzZkaxoLv/l2K8HaI0t5AVA+YYgUew==" }, + "https-proxy-agent": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-3.0.1.tgz", + "integrity": "sha512-+ML2Rbh6DAuee7d07tYGEKOEi2voWPUGan+ExdPbPW6Z3svq+JCqr0v8WmKPOkz1vOVykPCBSuobe7G8GJUtVg==", + "requires": { + "agent-base": "^4.3.0", + "debug": "^3.1.0" + }, + "dependencies": { + "debug": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", + "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "requires": { + "ms": "^2.1.1" + } + } + } + }, "iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -2942,7 +3062,6 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", - "dev": true, "requires": { "once": "^1.3.0", "wrappy": "1" @@ -2951,8 +3070,7 @@ "inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, "ini": { "version": "1.3.5", @@ -3699,6 +3817,11 @@ "picomatch": "^2.0.5" } }, + "mime": { + "version": "2.4.4", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz", + "integrity": "sha512-LRxmNwziLPT828z+4YkNzloCFC2YM4wrB99k+AV5ZbEyfGNWfG8SO1FUXLmLDBSo89NrJZ4DIWeLjy1CHGhMGA==" + }, "mime-db": { "version": "1.40.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.40.0.tgz", @@ -3729,7 +3852,6 @@ "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "dev": true, "requires": { "brace-expansion": "^1.1.7" } @@ -3762,7 +3884,6 @@ "version": "0.5.1", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", - "dev": true, "requires": { "minimist": "0.0.8" }, @@ -3770,8 +3891,7 @@ "minimist": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", - "dev": true + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" } } }, @@ -3784,8 +3904,7 @@ "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "mute-stream": { "version": "0.0.8", @@ -4310,8 +4429,7 @@ "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", - "dev": true + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" }, "path-is-inside": { "version": "1.0.2", @@ -4346,6 +4464,11 @@ "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", "dev": true }, + "pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=" + }, "picomatch": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.0.7.tgz", @@ -4485,14 +4608,17 @@ "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", - "dev": true + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, "progress": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", - "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", - "dev": true + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==" + }, + "proxy-from-env": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.0.0.tgz", + "integrity": "sha1-M8UDmPcOp+uW0h97gXYwpVeRx+4=" }, "proxyquire": { "version": "2.1.3", @@ -4526,6 +4652,21 @@ "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", "dev": true }, + "puppeteer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-2.0.0.tgz", + "integrity": "sha512-t3MmTWzQxPRP71teU6l0jX47PHXlc4Z52sQv4LJQSZLq1ttkKS2yGM3gaI57uQwZkNaoGd0+HPPMELZkcyhlqA==", + "requires": { + "debug": "^4.1.0", + "extract-zip": "^1.6.6", + "https-proxy-agent": "^3.0.0", + "mime": "^2.0.3", + "progress": "^2.0.1", + "proxy-from-env": "^1.0.0", + "rimraf": "^2.6.1", + "ws": "^6.1.0" + } + }, "quick-lru": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-1.1.0.tgz", @@ -4872,7 +5013,6 @@ "version": "2.7.1", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", - "dev": true, "requires": { "glob": "^7.1.3" } @@ -4904,8 +5044,7 @@ "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", - "dev": true + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" }, "safer-buffer": { "version": "2.1.2", @@ -5540,6 +5679,11 @@ "integrity": "sha512-DWkS49EQKVX//Tbupb9TFa19c7+MK1XmzkrZUR8TAktmE/DizXoaoJV6TZ/tSIPXipqNiRI6CyAe7x69Jb6RSw==", "dev": true }, + "typedarray": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", + "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" + }, "typedarray-to-buffer": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", @@ -5732,8 +5876,7 @@ "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", - "dev": true + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, "uuid": { "version": "3.3.3", @@ -5921,6 +6064,14 @@ "typedarray-to-buffer": "^3.1.5" } }, + "ws": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-6.2.1.tgz", + "integrity": "sha512-GIyAXC2cB7LjvpgMt9EKS2ldqr0MTrORaleiOno6TweZ6r3TKtoFQWay/2PceJ3RuBasOHzXNn5Lrw1X0bEjqA==", + "requires": { + "async-limiter": "~1.0.0" + } + }, "x-is-string": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/x-is-string/-/x-is-string-0.1.0.tgz", @@ -6046,6 +6197,14 @@ "requires": { "camelcase": "^4.1.0" } + }, + "yauzl": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.4.1.tgz", + "integrity": "sha1-lSj0QtqxsihOWLQ3m7GU4i4MQAU=", + "requires": { + "fd-slicer": "~1.0.1" + } } } } diff --git a/package.json b/package.json index bb2ed15..42fc0be 100644 --- a/package.json +++ b/package.json @@ -13,13 +13,15 @@ "description": "Package containing functions to calculated telemetry information.", "dependencies": { "applicationinsights": "^1.5.0", - "got": "^9.6.0" + "got": "^9.6.0", + "puppeteer": "^2.0.0" }, "devDependencies": { "@azure/functions": "^1.0.3", "@types/got": "^9.6.7", "@types/node": "^12.11.7", "@types/proxyquire": "^1.3.28", + "@types/puppeteer": "^1.20.2", "@types/sinon": "^7.5.0", "@typescript-eslint/eslint-plugin": "^2.5.0", "@typescript-eslint/parser": "^2.5.0", diff --git a/src/install-data/auth.ts b/src/install-data/auth.ts new file mode 100644 index 0000000..3a38729 --- /dev/null +++ b/src/install-data/auth.ts @@ -0,0 +1,42 @@ +import { Page } from 'puppeteer'; + +// TODO: Store in Azure Vault instead +const email = process.env.WEBHINT_STATS_EMAIL!; // eslint-disable-line no-process-env +const password = process.env.WEBHINT_STATS_PASSWORD!; // eslint-disable-line no-process-env +const chromeAuthUrl = 'https://chrome.google.com/webstore/developer/dashboard'; +const firefoxAuthUrl = 'https://addons.mozilla.org/en-US/developers/'; + +const delay = (ms: number) => { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +}; + +const signIn = async (page: Page) => { + await page.waitForSelector('input[type=password]'); + await page.type('input[type=email]', email); + await page.keyboard.press(String.fromCharCode(13)); + await delay(1000); + await page.waitForSelector('input[type=password]'); + await page.type('input[type=password]', password); + + await Promise.all([ + await page.keyboard.press(String.fromCharCode(13)), + await page.waitForNavigation() + ]); +}; + +export const signInChrome = async (page: Page) => { + await page.goto(chromeAuthUrl); + await signIn(page); +}; + +export const signInFirefox = async (page: Page) => { + await page.goto(firefoxAuthUrl); + await page.click('.DevHub-Navigation-Open'); + await Promise.all([ + page.click('.DevHub-Navigation-Register .Button'), + page.waitForNavigation() + ]); + await signIn(page); +}; diff --git a/src/install-data/chrome.ts b/src/install-data/chrome.ts new file mode 100644 index 0000000..acccee2 --- /dev/null +++ b/src/install-data/chrome.ts @@ -0,0 +1,40 @@ +import { BrowserInstallHistory } from './stats'; + +export type ChromeValue = { + v: T; +}; + +export type ChromeInstalls = { + c: [ + /** Date with zero-based month (e.g. in `"Date(2019,7,1)"` `7` = August) */ + ChromeValue, + /** Impressions */ + ChromeValue, + /** Installs */ + ChromeValue + ]; +}; + +export type ChromeInstallHistory = { + table: { + rows: ChromeInstalls[]; + }; +}; + +export const normalizeChromeInstalls = (chromeInstallHistory: ChromeInstallHistory) => { + const browserInstallHistory: BrowserInstallHistory = {}; + + for (const chromeInstalls of chromeInstallHistory.table.rows) { + // e.g. `[2019, 7, 1]` + const dateParts: [number, number, number] = chromeInstalls.c[0].v + .replace(/Date\(|\)/g, '') + .split(',') + .map(Number) as any; + + const dateString = new Date(Date.UTC(...dateParts)).toISOString(); + + browserInstallHistory[dateString] = chromeInstalls.c[2].v; + } + + return browserInstallHistory; +}; diff --git a/src/install-data/firefox.ts b/src/install-data/firefox.ts new file mode 100644 index 0000000..a16e981 --- /dev/null +++ b/src/install-data/firefox.ts @@ -0,0 +1,18 @@ +import { BrowserInstallHistory } from './stats'; + +export type FirefoxInstalls = { + count: number; + date: string; +}; + +export const normalizeFirefoxInstalls = (firefoxInstallHistory: FirefoxInstalls[]) => { + const browserInstallHistory: BrowserInstallHistory = {}; + + for (const firefoxInstalls of firefoxInstallHistory) { + const dateString = new Date(`${firefoxInstalls.date}Z`).toISOString(); + + browserInstallHistory[dateString] = firefoxInstalls.count; + } + + return browserInstallHistory; +}; diff --git a/src/install-data/index.ts b/src/install-data/index.ts new file mode 100644 index 0000000..a90f16c --- /dev/null +++ b/src/install-data/index.ts @@ -0,0 +1,9 @@ +import { AzureFunction, Context } from '@azure/functions'; + +import { getStats } from './stats'; + +const processInstallData: AzureFunction = async function (context: Context, myTimer: any): Promise { + await getStats(); +}; + +export default processInstallData; diff --git a/src/install-data/stats.ts b/src/install-data/stats.ts new file mode 100644 index 0000000..20c945b --- /dev/null +++ b/src/install-data/stats.ts @@ -0,0 +1,45 @@ +import { launch, Page } from 'puppeteer'; + +import { signInChrome, signInFirefox } from './auth'; + +export type BrowserInstallHistory = { [date: string]: number }; + +const today = new Date() + .toISOString() + .split('T')[0]; +const queryEnd = today.replace(/-/g, ''); +const chromeStatsUrl = 'https://chrome.google.com/webstore/developer/data?tq=base_stats%3Agccemnpihkbgkdmoogenkbkckppadcag&tqx=out%3Ajson'; +const firefoxStatsUrl = `https://addons.mozilla.org/en-US/firefox/addon/webhint/statistics/downloads-day-20190501-${queryEnd}.json`; + +const getBrowserStats = async (page: Page, url: string) => { + page.goto(url); + + const response = await page.waitForResponse(url); + + return await response.json(); +}; + +const getChromeStats = async (page: Page) => { + await signInChrome(page); + + return getBrowserStats(page, chromeStatsUrl); +}; + +const getFirefoxStats = async (page: Page) => { + await signInFirefox(page); + + return getBrowserStats(page, firefoxStatsUrl); +}; + +export const getStats = async () => { + const browser = await launch({ headless: false }); + const pages = await browser.pages(); + const page = pages[0]; + + const chrome = await getChromeStats(page); + const firefox = await getFirefoxStats(page); + + await browser.close(); + + return { chrome, firefox }; +}; diff --git a/tests/install-data/chrome.ts b/tests/install-data/chrome.ts new file mode 100644 index 0000000..bd71b81 --- /dev/null +++ b/tests/install-data/chrome.ts @@ -0,0 +1,24 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import test from 'ava'; + +import { normalizeChromeInstalls } from '../../src/install-data/chrome'; + +const rawStats = JSON.parse(fs.readFileSync(path.join(__dirname, 'fixtures', 'chrome.json'), 'utf-8')); // eslint-disable-line no-sync + +test('It can normalize install data from Chrome', (t) => { + const stats = normalizeChromeInstalls(rawStats); + + t.deepEqual(stats, { + '2019-10-01T00:00:00.000Z': 51, + '2019-10-02T00:00:00.000Z': 32, + '2019-10-03T00:00:00.000Z': 102, + '2019-10-04T00:00:00.000Z': 46, + '2019-10-05T00:00:00.000Z': 7, + '2019-10-06T00:00:00.000Z': 19, + '2019-10-07T00:00:00.000Z': 29, + '2019-10-08T00:00:00.000Z': 35, + '2019-10-09T00:00:00.000Z': 23, + '2019-10-10T00:00:00.000Z': 21 + }); +}); diff --git a/tests/install-data/firefox.ts b/tests/install-data/firefox.ts new file mode 100644 index 0000000..139d5d8 --- /dev/null +++ b/tests/install-data/firefox.ts @@ -0,0 +1,24 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import test from 'ava'; + +import { normalizeFirefoxInstalls } from '../../src/install-data/firefox'; + +const rawStats = JSON.parse(fs.readFileSync(path.join(__dirname, 'fixtures', 'firefox.json'), 'utf-8')); // eslint-disable-line no-sync + +test('It can normalize install data from Firefox', (t) => { + const stats = normalizeFirefoxInstalls(rawStats); + + t.deepEqual(stats, { + '2019-10-01T00:00:00.000Z': 147, + '2019-10-02T00:00:00.000Z': 128, + '2019-10-03T00:00:00.000Z': 159, + '2019-10-04T00:00:00.000Z': 85, + '2019-10-05T00:00:00.000Z': 47, + '2019-10-06T00:00:00.000Z': 30, + '2019-10-07T00:00:00.000Z': 79, + '2019-10-08T00:00:00.000Z': 195, + '2019-10-09T00:00:00.000Z': 100, + '2019-10-10T00:00:00.000Z': 112 + }); +}); diff --git a/tests/install-data/fixtures/chrome.json b/tests/install-data/fixtures/chrome.json new file mode 100644 index 0000000..c0aee79 --- /dev/null +++ b/tests/install-data/fixtures/chrome.json @@ -0,0 +1,169 @@ +{ + "version": "0.6", + "status": "ok", + "sig": "810059606", + "table": { + "cols": [ + { + "id": "date", + "label": "Date", + "type": "date", + "pattern": "" + }, + { + "id": "gccemnpihkbgkdmoogenkbkckppadcagImpressionsAll", + "label": "Impressions", + "type": "number", + "pattern": "", + "p": { + "item": "webhint", + "metric": "Impressions", + "source": "All Sources" + } + }, + { + "id": "gccemnpihkbgkdmoogenkbkckppadcagInstallationsAll", + "label": "Installations", + "type": "number", + "pattern": "", + "p": { + "item": "webhint", + "metric": "Installations", + "source": "All Sources" + } + } + ], + "rows": [ + { + "c": [ + { + "v": "Date(2019,9,1)" + }, + { + "v": 41 + }, + { + "v": 51 + } + ] + }, + { + "c": [ + { + "v": "Date(2019,9,2)" + }, + { + "v": 18 + }, + { + "v": 32 + } + ] + }, + { + "c": [ + { + "v": "Date(2019,9,3)" + }, + { + "v": 84 + }, + { + "v": 102 + } + ] + }, + { + "c": [ + { + "v": "Date(2019,9,4)" + }, + { + "v": 34 + }, + { + "v": 46 + } + ] + }, + { + "c": [ + { + "v": "Date(2019,9,5)" + }, + { + "v": 12 + }, + { + "v": 7 + } + ] + }, + { + "c": [ + { + "v": "Date(2019,9,6)" + }, + { + "v": 18 + }, + { + "v": 19 + } + ] + }, + { + "c": [ + { + "v": "Date(2019,9,7)" + }, + { + "v": 33 + }, + { + "v": 29 + } + ] + }, + { + "c": [ + { + "v": "Date(2019,9,8)" + }, + { + "v": 45 + }, + { + "v": 35 + } + ] + }, + { + "c": [ + { + "v": "Date(2019,9,9)" + }, + { + "v": 84 + }, + { + "v": 23 + } + ] + }, + { + "c": [ + { + "v": "Date(2019,9,10)" + }, + { + "v": 83 + }, + { + "v": 21 + } + ] + } + ] + } +} \ No newline at end of file diff --git a/tests/install-data/fixtures/firefox.json b/tests/install-data/fixtures/firefox.json new file mode 100644 index 0000000..a1d4e11 --- /dev/null +++ b/tests/install-data/fixtures/firefox.json @@ -0,0 +1,52 @@ +[ + { + "count": 112, + "date": "2019-10-10", + "end": "2019-10-10" + }, + { + "count": 100, + "date": "2019-10-09", + "end": "2019-10-09" + }, + { + "count": 195, + "date": "2019-10-08", + "end": "2019-10-08" + }, + { + "count": 79, + "date": "2019-10-07", + "end": "2019-10-07" + }, + { + "count": 30, + "date": "2019-10-06", + "end": "2019-10-06" + }, + { + "count": 47, + "date": "2019-10-05", + "end": "2019-10-05" + }, + { + "count": 85, + "date": "2019-10-04", + "end": "2019-10-04" + }, + { + "count": 159, + "date": "2019-10-03", + "end": "2019-10-03" + }, + { + "count": 128, + "date": "2019-10-02", + "end": "2019-10-02" + }, + { + "count": 147, + "date": "2019-10-01", + "end": "2019-10-01" + } +] \ No newline at end of file