Merge pull request #31 from hpicgs/dev

Release v0.6.0

Author: Jasperhino

.github/workflows/main.yml

@@ -15,13 +15,13 @@ jobs:
 
     steps:
       - name: Checkout 🛎️
-        uses: 'actions/checkout@v3'
+        uses: "actions/checkout@v3"
       - name: Bump version and push tag 🏷
         uses: anothrNick/github-tag-action@1.36.0
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           WITH_V: true
-       
+
   deploy:
     name: Deploy HiViSer Frontend to GitHub Pages 🚀
     concurrency: ci-${{ github.ref }} # Recommended if you intend to make multiple deployments in quick succession.
@@ -40,9 +40,17 @@ jobs:
         uses: actions/setup-node@v2
         with:
           node-version: 16
-          cache: 'pnpm'
+          cache: "pnpm"
           cache-dependency-path: frontend/pnpm-lock.yaml
 
+      - name: Install treemaps library 📦
+        run: |
+          cd frontend
+          git clone --branch hiviser_deployment https://${{ secrets.SEERENE_TREEMAPS_DEPLOY_USER }}:${{ secrets.SEERENE_TREEMAPS_DEPLOY_TOKEN }}@gitlab.hpi3d.de/seerene/treemaps.git
+          cd treemaps
+          npm install
+          npm run build-lib
+
       - name: Install and Build 🔧
         run: |
           cd frontend

.github/workflows/pull-request.yml

@@ -32,7 +32,7 @@ jobs:
           cd frontend
           git clone --branch hiviser_deployment https://${{ secrets.SEERENE_TREEMAPS_DEPLOY_USER }}:${{ secrets.SEERENE_TREEMAPS_DEPLOY_TOKEN }}@gitlab.hpi3d.de/seerene/treemaps.git
           cd treemaps
-          npm i
+          npm install
           npm run build-lib
 
       - name: Install and Build 🔧

.github/workflows/test.yml

@@ -19,4 +19,6 @@ jobs:
       - name: Run HiViSer Action 🚀
         uses: ./ # Uses the action in the root directory
         with:
-          repository_path: ./testrepo
+          benchmark: true
+          # repository_path: ./testrepo
+

README.md

@@ -2,27 +2,24 @@
 Seminar Project of the Seminar "Advanced Techniques for Analysis and Visualization of Software Data" of CGS, HPI and DEF in the Summer Term 2022
 
 # Usage
-⚠ This will not work until we made our first release
-
-Create a new GitHub Actions workflow in your project, e.g. at .github/workflows/hiviser.yml. The content of the file should be in the following format:
+Create a new GitHub Actions workflow in your project, e.g. at `.github/workflows/analytics-embedding.yml`. The content of the file should be in the following format:
 ```yaml
-name: HiViSer
+name: Analytics Treemap Embedding
 
 on:
   # Trigger the workflow on push or pull request,
   # but only for the main branch
   push:
     branches:
       - main
-  # Replace pull_request with pull_request_target if you
-  # plan to use this action with forks, see the Limitations section
+      
   pull_request:
     branches:
       - main
 
 jobs:
-  run-hiviser:
-    name: Run HiViSer
+  analytics-embedding:
+    name: Run Analytics Treemap Embedding 🔎
     runs-on: ubuntu-latest
 
     steps:
@@ -31,9 +28,11 @@ jobs:
 
       - name: Run Hiviser Action
         uses: hpicgs/hiviser-action@v0
-        # Optional, use if you want to analyse a specific folder
         with:
+          # Optional, use if you want to analyse a specific folder
           repository_path: ./
+          # Optional to enable benchmarking
+          benchmark: true
 ```
 
 ## Development

action.yml

@@ -18,8 +18,13 @@ inputs:
     description: >
       The root directory of the repository.
     default: "."
+  benchmark:
+    description: >
+      Enable benchmarking to time the different steps of the action.
+    default: "false"
 runs:
   using: "docker"
   image: "Dockerfile"
   env:
     REPOSITORY_PATH: ${{ inputs.repository_path }}
+    BENCHMARK: ${{ inputs.benchmark }}

analytics/src/github.ts

@@ -71,7 +71,7 @@ async function createBlob(content: string) {
   console.log(response);
 }
 
-async function createTree(csv: string, json: string): Promise<string> {
+async function createTree(csv: string): Promise<string> {
   console.log(`creating tree at ${owner}/${repo}`);
 
   const response = await octokit.request(
@@ -86,12 +86,6 @@ async function createTree(csv: string, json: string): Promise<string> {
           type: "blob",
           content: csv,
         },
-        {
-          path: "metrics.json",
-          mode: "100644",
-          type: "blob",
-          content: json,
-        },
       ],
     }
   );
@@ -100,15 +94,12 @@ async function createTree(csv: string, json: string): Promise<string> {
   return response.data.sha;
 }
 
-export async function storeMetricsToRepo(
-  metrics_csv: string,
-  metrics_json: string
-) {
+export async function storeMetricsToRepo(metrics_csv: string) {
   if (process.env.DEBUG) {
     console.log("DEBUG mode enabled, skipping GitHub API calls");
     return;
   }
 
-  const tree_sha = await createTree(metrics_csv, metrics_json);
+  const tree_sha = await createTree(metrics_csv);
   await createRef(`refs/metrics/${commit_sha}`, tree_sha);
 }

analytics/src/index.ts

@@ -1,21 +1,14 @@
-import { lstatSync, readdirSync, Stats, statSync, writeFileSync } from "fs";
-import { promisify } from "util";
 import glob from "glob";
+import { writeFileSync } from "fs";
+import { promisify } from "util";
 import { storeMetricsToRepo } from "./github";
-import { basename, extname, join } from "path";
 import { createObjectCsvStringifier } from "csv-writer";
 import { getMetrics } from "./metrics";
-import {
-  DirectoryNode,
-  FileNode,
-  Metrics,
-  MetricsNode,
-  NodeType,
-} from "./types";
+import { performance } from "perf_hooks";
 
 const globPromise = promisify(glob);
 
-async function analyseRepository() {
+async function analyseRepository(benchmark = false) {
   if (!process.env.REPOSITORY_PATH)
     throw new Error("REPOSITORY_PATH environment variable is not set");
   console.log(`REPOSITORY_PATH="${process.env.REPOSITORY_PATH}"`);
@@ -26,130 +19,49 @@ async function analyseRepository() {
   const csvStringifier = createObjectCsvStringifier({
     header: [
       { id: "filename", title: "filename" },
-      { id: "loc", title: "Line of Code" },
-      { id: "noc", title: "Number of Comments" },
-      { id: "cloc", title: "Commented Lines of Code" },
-      { id: "dc", title: "Density of Comments" },
-      { id: "nof", title: "Number of Functions" },
+      { id: "loc", title: "loc" }, // Lines of Code
+      { id: "noc", title: "noc" }, // Number of Comments
+      { id: "cloc", title: "cloc" }, //Commented Lines of Code
+      { id: "dc", title: "dc" }, // Density of Comments
+      { id: "nof", title: "nof" }, // Number of Functions
     ],
   });
 
-  const metrics = files.map((filename) => ({
-    filename,
-    ...getMetrics(filename),
-  }));
+  let timings: { [filename: string]: number } = {};
+  if (benchmark) console.time("calculating-metrics");
+  const metrics = files.map((filename) => {
+    if (benchmark) {
+      const start = performance.now();
+      const m = {
+        filename,
+        ...getMetrics(filename),
+      };
+      const end = performance.now();
+      timings[filename] = end - start;
+      return m;
+    } else {
+      return {
+        filename,
+        ...getMetrics(filename),
+      };
+    }
+  });
+  if (benchmark) console.timeEnd("calculating-metrics");
+  if (benchmark) console.log(timings);
+  if (benchmark) console.time("writing-csv");
+
   const header = csvStringifier.getHeaderString();
   const csv = header + csvStringifier.stringifyRecords(metrics);
-  const metricsTree = parseRepositoryTree(process.env.REPOSITORY_PATH);
-  const json = JSON.stringify(metricsTree, null, 2);
-
   writeFileSync("metrics.csv", csv);
-  writeFileSync("metrics.json", json);
-
-  storeMetricsToRepo(csv, json);
-}
-
-// Adopted from https://github.com/mihneadb/node-directory-tree/blob/master/lib/directory-tree.js#L21
-function safeReadDirSync(path: string) {
-  let dirData = [];
-  try {
-    dirData = readdirSync(path);
-  } catch (ex: any) {
-    if (ex.code == "EACCES" || ex.code == "EPERM") {
-      //User does not have permissions, ignore directory
-      return null;
-    } else throw ex;
-  }
-  return dirData;
-}
-
-function reduceMetrics(children: MetricsNode[]): Metrics {
-  const metricsList = children.reduce(
-    (result: Metrics[], element: MetricsNode) => {
-      if (element.metrics) {
-        result.push(element.metrics);
-      }
-      return result;
-    },
-    []
-  );
 
-  return metricsList.reduce(
-    (acc, cur) => ({
-      loc: acc.loc + cur.loc,
-      noc: acc.noc + cur.noc,
-      cloc: acc.cloc + cur.cloc,
-      //Todo this should not be accumulated
-      dc: acc.dc + cur.dc,
-      nof: acc.nof + cur.nof,
-    }),
-    { loc: 0, noc: 0, cloc: 0, dc: 0, nof: 0 }
-  );
-}
-
-function parseRepositoryTree(
-  path: string,
-  currentDepth = 0,
-  maxDepth: number | undefined = undefined
-): MetricsNode | null {
-  const extensions = /\.ts$/;
-  const name = basename(path);
-
-  let stats: Stats;
-  let lstats: Stats;
-
-  try {
-    stats = statSync(path);
-    lstats = lstatSync(path);
-  } catch (e) {
-    return null;
-  }
-
-  if (lstats.isSymbolicLink()) {
-    // Skip symbolic links
-    return null;
-  }
-
-  if (stats.isFile()) {
-    const ext = extname(path).toLowerCase();
-
-    // Skip if it does not match the extension regex
-    if (extensions && !extensions.test(ext)) return null;
-    const fileNode = {
-      path,
-      name,
-      type: NodeType.FILE,
-      metrics: getMetrics(path),
-    } as FileNode;
-    return fileNode;
-  } else if (stats.isDirectory()) {
-    const dirNode = { path, name, type: NodeType.DIRECTORY } as DirectoryNode;
-    let dirData = safeReadDirSync(path);
-    if (dirData === null) return null;
-
-    if (maxDepth === undefined || maxDepth > currentDepth) {
-      dirNode.children = dirData.reduce(
-        (children: MetricsNode[], childPath: string) => {
-          const child = parseRepositoryTree(
-            join(path, childPath),
-            currentDepth + 1
-          );
-          if (child) {
-            children.push(child);
-          }
-          return children;
-        },
-        []
-      );
-
-      dirNode.metrics = reduceMetrics(dirNode.children);
-    }
-    return dirNode;
-  }
-  return null;
+  if (benchmark) console.timeEnd("writing-csv");
+  if (benchmark) console.time("storing-in-github");
+  await storeMetricsToRepo(csv);
+  if (benchmark) console.timeEnd("storing-in-github");
 }
 
-analyseRepository().catch((e) => {
+const benchmark = Boolean(process.env.BENCHMARK);
+analyseRepository(benchmark).catch((e) => {
   console.error(e);
   process.exit(1);
 });

analytics/src/types.ts

@@ -1,15 +1,5 @@
-export interface FileMetrics {
-  [key: string]: any;
-}
-
-export interface MetricsTableData {
-  header: string[];
-  rows: FileMetrics[];
-}
-
-export enum NodeType {
-  FILE = "file",
-  DIRECTORY = "directory",
+export interface FileMetrics extends Metrics {
+  filename: string;
 }
 
 export interface Metrics {
@@ -19,19 +9,17 @@ export interface Metrics {
   dc: number;
   nof: number;
 }
-
-export interface MetricsNode {
-  path: string;
-  name: string;
-  type: NodeType;
-  metrics: Metrics;
+export interface MetricsTableData {
+  header: string[];
+  rows: FileMetrics[];
 }
 
-export interface DirectoryNode extends MetricsNode {
-  type: NodeType.DIRECTORY;
-  children: MetricsNode[];
+export interface FileTree {
+  root: TreeNode;
 }
 
-export interface FileNode extends MetricsNode {
-  type: NodeType.FILE;
+export interface TreeNode {
+  name: string;
+  children: TreeNode[];
+  metrics?: Metrics;
 }