feat: change validate action to take into consideration only add… (#422)

`node scripts/data-valdiate.js` cand also be run locally

This also will fix the issue when some valid URL returned timeout
Because the action wanted to make all requests at the same time

`git restore` is available only from Git 2.23

https://github.blog/2019-08-16-highlights-from-git-2-23/#experimental-alternatives-for-git-checkout

Closes #382

.github/workflows/data-validate.yml

@@ -15,11 +15,15 @@ jobs:
     - uses: actions/setup-node@v1
       with:
         node-version: 13.x
-    - name: Install validation libs
+
-      run: |
+    - name: Cache/Restore node modules
-        npm install -g @hapi/joi@17.0.2
+      uses: actions/cache@v1
-        npm install -g @actions/core@1.2.0
+      with:
-        npm link @hapi/joi
+        path: ~/.npm # npm cache files are stored in `~/.npm` on Linux/macOS
-        npm link @actions/core
+        key: ${{ runner.os }}-npm-${{ hashFiles('./package-lock.json') }}
+
+    - name: Install Dependencies
+      run: npm install
+
     - name: Validate data.js
-      run: node ./scripts/data-validate.js
+      run: node ./scripts/data-validate.js

package-lock.json

@@ -5,9 +5,22 @@
   "requires": true,
   "dependencies": {
     "@actions/core": {
-      "version": "1.2.0",
+      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.2.0.tgz",
+      "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.2.1.tgz",
-      "integrity": "sha512-ZKdyhlSlyz38S6YFfPnyNgCDZuAF2T0Qv5eHflNWytPS8Qjvz39bZFMry9Bb/dpSnqWcNeav5yM2CTYpJeY+Dw=="
+      "integrity": "sha512-xD+CQd9p4lU7ZfRqmUcbJpqR+Ss51rJRVeXMyOLrZQImN9/8Sy/BEUBnHO/UKD3z03R686PVTLfEPmkropGuLw=="
+    },
+    "@actions/exec": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/@actions/exec/-/exec-1.0.3.tgz",
+      "integrity": "sha512-TogJGnueOmM7ntCi0ASTUj4LapRRtDfj57Ja4IhPmg2fls28uVOPbAn8N+JifaOumN2UG3oEO/Ixek2A4NcYSA==",
+      "requires": {
+        "@actions/io": "^1.0.1"
+      }
+    },
+    "@actions/io": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/@actions/io/-/io-1.0.2.tgz",
+      "integrity": "sha512-J8KuFqVPr3p6U8W93DOXlXW6zFvrQAJANdS+vw0YhusLIq+bszW8zmK2Fh1C2kDPX8FMvwIl1OUcFgvJoXLbAg=="
     },
     "@babel/code-frame": {
       "version": "7.5.5",

package.json

@@ -10,7 +10,8 @@
     ]
   },
   "dependencies": {
-    "@actions/core": "^1.2.0",
+    "@actions/core": "^1.2.1",
+    "@actions/exec": "^1.0.3",
     "@hapi/joi": "^17.0.2",
     "country-emoji": "^1.5.0",
     "esm": "^3.2.25",

scripts/data-validate.js

@@ -1,88 +1,42 @@
-import Joi from '@hapi/joi';
 import core from '@actions/core';
-import * as http from 'http';
+import { getMasterData, Schema, getStatusCode } from './utils.js';
-import * as https from 'https';
+import srcData from '../src/data.js';
-import data from '../src/data.js';
-import flags from './flags.js';
-
-if (process.env.CI !== 'true') {
-  core.error = console.error;
-  core.setFailed = console.error;
-}
-
-const schema = Joi.object({
-  name: Joi.string().required(),
-  description: Joi.string().required(),
-  url: Joi.string()
-    .uri()
-    .required()
-    .pattern(/(use|uses|using|setup|environment|^https:\/\/gist.github.com\/)/),
-  country: Joi.string()
-    .valid(...flags)
-    .required(),
-  twitter: Joi.string().pattern(new RegExp(/^@?(\w){1,15}$/)),
-  emoji: Joi.string().allow(''),
-  computer: Joi.string().valid('apple', 'windows', 'linux'),
-  phone: Joi.string().valid('iphone', 'android'),
-  tags: Joi.array().items(Joi.string()),
-});
-
-const errors = data
-  .map(person => schema.validate(person))
-  .filter(v => v.error)
-  .map(v => v.error);
-
-errors.forEach(e => {
-  core.error(e._original.name);
-  e.details.forEach(d => core.error(d.message));
-});
-
-if (errors.length) {
-  core.setFailed('Action failed with validation errors, see logs');
-}
-const REQUEST_TIMEOUT = 10000;
-
-function getStatusCode(url) {
-  const client = url.startsWith('https') ? https : http;
-  return new Promise((resolve, reject) => {
-    setTimeout(() => reject(new Error('Request timed out')), REQUEST_TIMEOUT);
-    client
-      .get(url, res => {
-        resolve(res.statusCode);
-      })
-      .on('error', err => {
-        reject(err);
-      });
-  });
-}
-
-async function isWorkingUrl(url) {
-  try {
-    const statusCode = await getStatusCode(url);
-    if (statusCode < 200 || statusCode >= 400) {
-      core.error(`Ping to "${url}" failed with status: ${statusCode}`);
-      return false;
-    }
-    return true;
-  } catch (e) {
-    core.error(`Ping to "${url}" failed with error: ${e}`);
-    return false;
-  }
-}
 
 (async () => {
-  // TODO: we might need to batch these in sets instead of requesting 100+ URLs
+  // on master branch will be empty array
-  // at the same time
+  const masterDataUrls = (await getMasterData()).map(d => d.url);
-  const areWorkingUrls = await Promise.all(
+  // so here data will be an array with all users
-    data.map(p => p.url).map(url => isWorkingUrl(url))
+  const data = srcData.filter(d => !masterDataUrls.includes(d.url));
-  );
+
-  const failingUrls = areWorkingUrls.filter(a => !a);
+  const errors = data
-  if (failingUrls.length > 0) {
+    .map(person => Schema.validate(person))
-    core.setFailed(
+    .filter(v => v.error)
-      `Action failed with ${failingUrls.length} URL fetch failures, see logs`
+    .map(v => v.error);
-    );
+
+  errors.forEach(e => {
+    core.error(e._original.name || e._original.url);
+    e.details.forEach(d => core.error(d.message));
+  });
+
+  let failedUrlsCount = 0;
+  for await (const { url } of data) {
+    try {
+      const statusCode = await getStatusCode(url);
+      if (statusCode < 200 || statusCode >= 400) {
+        core.error(`Ping to "${url}" failed with status: ${statusCode}`);
+        failedUrlsCount += 1;
+      }
+    } catch (e) {
+      core.error(`Ping to "${url}" failed with error: ${e}`);
+      failedUrlsCount += 1;
+    }
   }
-  if (process.env.CI !== 'true') {
+
-    process.exit(failingUrls.length > 0 ? 1 : 0)
+  if (failedUrlsCount) {
+    core.error(`Action failed with ${failedUrlsCount} URL fetch failures`);
+  }
+
+  if (errors.length || failedUrlsCount) {
+    core.setFailed('Action failed with errors, see logs');
   }
 })();

scripts/masterData.js

@@ -0,0 +1,5 @@
+/**
+ * this is a stub file, do not edit it
+ * see `scripts/utils.js` -> `getMasterData`
+ */
+export default [];

scripts/utils.js

@@ -0,0 +1,85 @@
+import exec from '@actions/exec';
+import core from '@actions/core';
+import Joi from '@hapi/joi';
+import * as http from 'http';
+import * as https from 'https';
+import flags from './flags.js';
+
+async function getCurrentBranchName() {
+  let myOutput = '';
+  let myError = '';
+
+  const options = {
+    silent: true,
+    listeners: {
+      stdout: data => (myOutput += data.toString()),
+      stderr: data => (myError += data.toString()),
+    },
+  };
+
+  await exec.exec('git rev-parse --abbrev-ref HEAD', [], options);
+  return myOutput.trim();
+}
+
+/** on master branch will return an empty array */
+export async function getMasterData() {
+  const options = { silent: true };
+  const curentBranchName = getCurrentBranchName();
+  // when on a branch/PR different from master
+  // will populate scripts/masterData.js with src/data.js from master
+  if (curentBranchName !== 'master') {
+    core.info('Executing action on branch different from master');
+    await exec.exec('mv src/data.js src/tmpData.js', [], options);
+    await exec.exec('git fetch origin master', [], options);
+    await exec.exec('git restore --source=FETCH_HEAD src/data.js', [], options);
+    await exec.exec('mv src/data.js scripts/masterData.js', [], options);
+    await exec.exec('mv src/tmpData.js src/data.js', [], options);
+  } else {
+    core.info('Executing action on master branch');
+  }
+
+  const masterData = await import('./masterData.js').then(m => m.default);
+
+  // restore `scripts/masterData.js` after was loaded
+  if (curentBranchName !== 'master') {
+    await exec.exec('git restore scripts/masterData.js', [], options);
+  }
+
+  return masterData;
+}
+
+export const Schema = Joi.object({
+  name: Joi.string().required(),
+  description: Joi.string().required(),
+  url: Joi.string()
+    .uri()
+    .required()
+    .pattern(/(use|uses|using|setup|environment|^https:\/\/gist.github.com\/)/),
+  country: Joi.string()
+    .valid(...flags)
+    .required(),
+  twitter: Joi.string().pattern(new RegExp(/^@?(\w){1,15}$/)),
+  emoji: Joi.string().allow(''),
+  computer: Joi.string().valid('apple', 'windows', 'linux'),
+  phone: Joi.string().valid('iphone', 'android'),
+  tags: Joi.array().items(Joi.string()),
+});
+
+export function getStatusCode(url) {
+  const client = url.startsWith('https') ? https : http;
+  return new Promise((resolve, reject) => {
+    const REQUEST_TIMEOUT = 10000;
+    const timeoutId = setTimeout(
+      reject,
+      REQUEST_TIMEOUT,
+      new Error('Request timed out')
+    );
+
+    client
+      .get(url, res => {
+        clearTimeout(timeoutId);
+        resolve(res.statusCode);
+      })
+      .on('error', err => reject(err));
+  });
+}