Add visualization feature for fork activity

Dockerfile.client

@@ -5,6 +5,8 @@ COPY client/package.json /usr/src/app
 
 WORKDIR /usr/src/app
 RUN npm install --legacy-peer-deps
+RUN npm install d3 --legacy-peer-deps
+RUN npm install react-select --legacy-peer-deps
 
 COPY client /usr/src/app
 RUN npm run build

app/__init__.py

@@ -13,6 +13,7 @@
 from .api.FollowRepository import FollowRepository
 from .api.Auth import Auth
 from .api.ForkList import ForkList
+from .api.Progress import Progress
 from .api.ForkClustering import ForkClustering
 from .db import initialize_db
 from .loginmanager import login_manager
@@ -105,6 +106,12 @@ def serve(path):
         resource_class_kwargs={"jwt": jwt},
     )
 
+    api.add_resource(
+        Progress,
+        "/flask/progress",
+        resource_class_kwargs={"jwt": jwt},
+    )
+
     # TODO: get correct host, broker and backend depending on environment
     redis_host = "redis://redis:6379/0"
     celery.conf.broker_url = redis_host

app/analyse/analyser.py

@@ -151,6 +151,38 @@ def get_commit_number_per_hour(repo, access_token):
     commit_info = res.json()
     return commit_info
 
+def get_commit_number_per_week(repo, access_token):
+
+    request_url = "https://api.github.com/repos/%s/stats/participation" % repo
+
+    res = requests.get(
+        url=request_url,
+        headers={
+            "Accept": "application/json",
+            "Authorization": "token {}".format(access_token),
+        },
+    )
+    commit_info = res.json()
+    print(commit_info)
+    if ('all' in commit_info.keys()):
+        return commit_info['all']
+    else:
+        return [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+
+def get_commit_number_per_hour(repo, access_token):
+
+    request_url = "https://api.github.com/repos/%s/stats/commit_activity" % repo
+
+    res = requests.get(
+        url=request_url,
+        headers={
+            "Accept": "application/json",
+            "Authorization": "token {}".format(access_token),
+        },
+    )
+    commit_info = res.json()
+    return commit_info
+
 @celery.task
 def start_analyse(repo, access_token):
     """Start analyse on repo using github_api_caller(contains personal access token)
@@ -186,7 +218,7 @@ def start_analyse(repo, access_token):
         current_app.config["LOCAL_DATA_PATH"] + "/" + repo + "/forks_list.json"
     )
 
-    active_forks = get_active_forks(repo,access_token )
+    active_forks = get_active_forks(repo, access_token)
 
     if current_app.config["USE_LOCAL_FORKS_LIST"] and os.path.exists(forks_list_path):
         with open(forks_list_path) as read_file:

app/analyse/project_updater.py

@@ -87,9 +87,9 @@ def work(self):
             if os.path.exists(self.diff_result_path):
                 with open(self.diff_result_path) as read_file:
                     compare_result = json.load(read_file)
-            else:
-                # local file not exist
-                return
+            # else:
+            #     # local file not exist
+            #     return
         else:
             # If the compare result is not crawled, start to crawl.
             splitForkName = self.fork_name.split("/")

app/api/FollowRepository.py

@@ -115,4 +115,5 @@ def post(self):
                 "timesForked": res["forks_count"],
                 "repo": res["full_name"],
             },
+            "analyser_progress": db_find_project(repo)["analyser_progress"]
         }

app/api/ForkList.py

@@ -198,7 +198,7 @@ def post(self):
         forks_info = ProjectFork.objects(project_name=repo)
         fork = forks_info[index]
         return_list = []
-
+        
         return_list.append(
             {
                 "fork_name": fork["fork_name"],

app/api/Progress.py

@@ -0,0 +1,71 @@
+from flask_restful import Resource
+from flask_jwt_extended import get_jwt_identity
+from flask_jwt_extended import jwt_required
+import json
+from flask import request
+import requests
+from ..models import User, ProjectFork, Project
+from ..analyse.compare_changes_crawler import fetch_commit_list, fetch_diff_code
+from ..analyse.analyser import get_active_forks
+from ..analyse.analyser import get_commit_number_per_week
+from ..analyse.analyser import get_commit_number_per_hour
+from rake_nltk import Rake
+
+def db_find_project(project_name):
+    return Project.objects(project_name=project_name).first()
+
+
+class Progress(Resource):
+    def __init__(self, jwt):
+        self.jwt = jwt
+
+    @jwt_required()
+    def post(self):
+
+        current_user = get_jwt_identity()
+        _user = User.objects(username=current_user).first()
+
+        req_data = request.get_json()
+        repoName = req_data.get("repo")
+        index = req_data.get("index")
+        repo = repoName
+
+        forks_info = ProjectFork.objects(project_name=repo)
+        fork = forks_info[index]
+        return_list = []
+
+        return_list.append(
+            {
+                "fork_name": fork["fork_name"],
+                "project_name": fork["project_name"],
+                "num_changed_files": fork["total_changed_file_number"],
+                "num_changed_lines": fork["total_changed_line_number"],
+                "changed_files": fork["file_list"],
+                "key_words": fork["key_words"],
+                "tags": fork["tags"],
+                "total_commit_number": fork["total_commit_number"],
+                "last_committed_time": str(fork["last_committed_time"]),
+                "created_time": str(fork["created_time"]),
+                "weekly_commit_freq": get_commit_number_per_week(fork["fork_name"],  _user.github_access_token),
+                "hourly_commit_freq": get_commit_number_per_hour(fork["fork_name"],  _user.github_access_token),
+            }
+        )
+
+        return {"forks": return_list}
+
+    @jwt_required()
+    def get(self):
+
+
+        current_user = get_jwt_identity()
+        _user = User.objects(username=current_user).first()
+
+        req_data = request.args
+        repoName = req_data.get("repo")
+        repo = repoName
+
+        forks_info = ProjectFork.objects(project_name=repo)
+
+        return len(forks_info)
+
+

client/src/App.js

@@ -22,6 +22,7 @@ import LoginModal from "./LoginModal";
 import ForkCluster from "./ForkCluster";
 import { getUserLogin } from "./repository";
 import Forklist from "./Forklist";
+import ForkGraph from "./ForkGraph";
 import DrawerCard from "./DrawerCard";
 
 const theme = createTheme({
@@ -163,6 +164,16 @@ const App = () => {
                 )
               }
             />
+            <Route
+              path="/visual/:repo1/:repo2"
+              element={
+                !isEmpty(currentUser) ? (
+                  <ForkGraph />
+                ) : (
+                  <Navigate to="/login" />
+                )
+              }
+            />
             <Route
               path="/forks/:repo1/:repo2"
               element={

client/src/FollowedRepositoryCard.jsx

@@ -14,6 +14,7 @@ import RemoveButton from "./common/RemoveButton";
 import { SECONDARY, TERTIARY } from "./common/constants";
 import { deleteUserRepository } from "./repository";
 import SignalCellularAltIcon from "@mui/icons-material/SignalCellularAlt";
+import Assessment from "@mui/icons-material/Assessment";
 import { useNavigate } from "react-router";
 import { PRIMARY, REMOVE } from "./common/constants";
 
@@ -42,19 +43,23 @@ const FollowedRepositoryCard = ({
     navigateToFork();
   };
 
+  const setForkGraph = () => {
+    console.log("repo nav", repo);
+    navigate(`/visual/${repo}`, { replace: true });
+  };
+
   return (
     <Box paddingY={1}>
       <Accordion>
         <AccordionSummary
           style={{ background: SECONDARY }}
           expandIcon={<ExpandMoreIcon sx={{ color: "white" }} />}
         >
-          <Grid container direction="row" alignItems="center">
-            <Grid item xs={9}>
+          <Grid container direction="row" alignItems="center" spacing={0.5}>
+            <Grid item xs={8.6}>
               <Typography color="white">{repo}</Typography>
             </Grid>
-            <Grid item xs={2}>
-              <Box style={{ textAlign: "right" }}>
+            <Grid item xs={1.2}>
                 <Button
                   color="inherit"
                   startIcon={<SignalCellularAltIcon />}
@@ -63,12 +68,19 @@ const FollowedRepositoryCard = ({
                 >
                   View Forks
                 </Button>
-              </Box>
+            </Grid>
+            <Grid item xs={1.2}>
+                <Button
+                  color="inherit"
+                  startIcon={<Assessment />}
+                  onClick={setForkGraph}
+                  style={{ background: REMOVE }}
+                >
+                  View Graph
+                </Button>
             </Grid>
             <Grid item xs={1}>
-              <Box style={{ textAlign: "center" }}>
                 <RemoveButton onClickRemove={onRemove} />
-              </Box>
             </Grid>
           </Grid>
         </AccordionSummary>