diff --git a/.env.example b/.env.example new file mode 100644 index 0000000000..b476b3d294 --- /dev/null +++ b/.env.example @@ -0,0 +1,11 @@ +GITHUB_TOKEN='' +GITHUB_REPOSITORY='DataKind-BLR/covid19bharat_data' +GITHUB_ACTOR='DataKind-BLR' +GITHUB_NAME='DataKind-BLR' +ORIGIN_BRANCH='origin' +MAIN_BRANCH='main' +GH_PAGES_BRANCH='gh-pages' +DEV_BRANCH='dev' +TEMP_DIR='tmp' +CODE_DIR='code' +MODE='dev/prod' \ No newline at end of file diff --git a/.github/workflows/javascript.yml b/.github/workflows/javascript.yml index 383943d199..2d428fb1bb 100644 --- a/.github/workflows/javascript.yml +++ b/.github/workflows/javascript.yml @@ -7,8 +7,6 @@ on: push: branches: - main - # repository_dispatch: - # types: [my-event] jobs: update-json-data: @@ -44,39 +42,6 @@ jobs: echo "the job completed" env: GITHUB_TOKEN: ${{ secrets.github_token }} - SHEET_ID_v1: ${{ secrets.SHEET_ID_v1 }} - SHEET_ID_v2: ${{ secrets.SHEET_ID_v2 }} - SHEET_ID_v3: ${{ secrets.SHEET_ID_v3 }} - SHEET_ID_v4: ${{ secrets.SHEET_ID_v4 }} - SHEET_ID_v5: ${{ secrets.SHEET_ID_v5 }} - SHEET_ID_v6: ${{ secrets.SHEET_ID_v6 }} - SHEET_ID_v7: ${{ secrets.SHEET_ID_v7 }} - SHEET_ID_v8: ${{ secrets.SHEET_ID_v8 }} - SHEET_ID_v9: ${{ secrets.SHEET_ID_v9 }} - SHEET_ID_v10: ${{ secrets.SHEET_ID_v10 }} - SHEET_ID_v11: ${{ secrets.SHEET_ID_v11 }} - SHEET_ID_v12: ${{ secrets.SHEET_ID_v12 }} - SHEET_ID_v13: ${{ secrets.SHEET_ID_v13 }} - SHEET_ID_v14: ${{ secrets.SHEET_ID_v14 }} - SHEET_ID_v15: ${{ secrets.SHEET_ID_v15 }} - SHEET_ID_v16: ${{ secrets.SHEET_ID_v16 }} - SHEET_ID_v17: ${{ secrets.SHEET_ID_v17 }} - SHEET_ID_v18: ${{ secrets.SHEET_ID_v18 }} - SHEET_ID_v19: ${{ secrets.SHEET_ID_v19 }} - SHEET_ID_v20: ${{ secrets.SHEET_ID_v20 }} - SHEET_ID_v21: ${{ secrets.SHEET_ID_v21 }} - SHEET_ID_v22: ${{ secrets.SHEET_ID_v22 }} - SHEET_ID_v23: ${{ secrets.SHEET_ID_v23 }} - SHEET_ID_v24: ${{ secrets.SHEET_ID_v24 }} - SHEET_ID_v25: ${{ secrets.SHEET_ID_v25 }} - SHEET_ID_v26: ${{ secrets.SHEET_ID_v26 }} - SHEET_ID_v27: ${{ secrets.SHEET_ID_v27 }} - SHEET_ID_v28: ${{ secrets.SHEET_ID_v28 }} - SHEET_ID_v29: ${{ secrets.SHEET_ID_v29 }} - SHEET_ID_v30: ${{ secrets.SHEET_ID_v30 }} - SHEET_ID_v31: ${{ secrets.SHEET_ID_v31 }} - SHEET_ID_v32: ${{ secrets.SHEET_ID_v32 }} - SHEET_ID_v33: ${{ secrets.SHEET_ID_v33 }} RESOURCES_SHEET_ID: ${{ secrets.RESOURCES_SHEET_ID }} SHEET_LOCALE: ${{ secrets.SHEET_LOCALE }} STUCK_BOT: ${{ secrets.STUCK_BOT }} @@ -89,16 +54,3 @@ jobs: status: ${{job.status}} STUCK_BOT: ${{ secrets.STUCK_BOT }} GITHUB_TOKEN: ${{ secrets.github_token }} - # - name: Repository Dispatch - # if: always() # ${{ !cancelled() }} - # uses: actions/github-script@v2 - # with: - # github-token: ${{secrets.JUN_TOKEN2}} - # script: | - # const fs = require('fs'); - # await github.repos.createDispatchEvent({ - # owner: context.repo.owner, - # repo: context.repo.repo, - # event_type: "my-event", - # client_payload: { greeting: "I'm Mr Meeseek. Loot at me!" } - # }); diff --git a/.gitignore b/.gitignore index 329fb3f4ca..002f954f50 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ bin-release/ # Other files and folders .settings/ +code/ # Executables *.swf @@ -40,3 +41,6 @@ tmp/ # Debug script fetch.sh +.DS_Store + +.env diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000000..676b665959 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,3 @@ +Head over to [covid19bharat_scrapers](https://github.com/DataKind-BLR/covid19bharat_scrapers) repository to understand how the data is being entered into the google sheets + +covid19bharat_data diff --git a/CNAME b/CNAME index f0e3cd0e54..6ff5782f80 100644 --- a/CNAME +++ b/CNAME @@ -1 +1 @@ -api.covid19india.org \ No newline at end of file +data.covid19bharat.org diff --git a/README.md b/README.md index 54d73bfb2a..951970d6ee 100644 --- a/README.md +++ b/README.md @@ -1,45 +1,44 @@ -# COVID19-India API +# COVID19-Bharat API ## Announcement -**We have stopped capturing testing data at a district level. Please check the status of the API endpoints below.** +**We are porting the data repository from [covid19india.org API](https://github.com/covid19india/data) to [covid19bharat.org API](https://github.com/DataKind-BLR/covid19bharat_data)** -1. As of 13th August our [API](https://github.com/covid19india/api) repository and api.covid19india.org have been deprecated. We redirect api.covid19india.org to data.covid19india.org -2. As of 13th August, all json endpoints except v4 endpoints have been deprecated. Please use the csv endpoints or refer to v4 endpoints +1. As of 31st October 2021, [data.covid19india.org](https://data.covid19india.org/) will stop updating the data. We will keep the data flowing on [data.covid19bharat.org](https://data.covid19bharat.org/) ### Files available - Aggregated sheets provide aggregated data at the district/state levels in csv format. -- V4 json endpoints. These are the json apis that are used by the website to show all the statistics on the site. These can be used by developers and analysts who have knowledge of json parsing (recommended approach). All our v4 endpoints are actively developed and in use since this serves the frontend view [Documentation for the same](https://api.covid19india.org/documentation). +- V4 json endpoints. These are the json apis that are used by the website to show all the statistics on the site. These can be used by developers and analysts who have knowledge of json parsing (recommended approach). All our v4 endpoints are actively developed and in use since this serves the frontend view [Documentation for the same](https://data.covid19bharat.org/documentation). - Latest data from the google sheet (10-20 minutes delayed) is available through the `latest` end-point. These are present under the `raw files` section below. (Not recommended since the number of files is huge and there is no additional information present in these as compared to the above mentioned endpoints.) ### V4 JSON endpoints | Status | Link to API | Description | | ------------- | -------------------------------------------------------- | ---------------------- | -| :green_heart: | | Daily numbers across C,R,D and Tested per state (historical data). | -| :green_heart: | | Current day numbers across districts and states. | +| :green_heart: | | Daily numbers across C,R,D and Tested per state (historical data). | +| :green_heart: | | Current day numbers across districts and states. | **Note**: Please consider using the above endpoints for all your data needs. All the data we show on the website is fuelled by the above endpoints. #### Aggregated Sheets (CSV) -| Status | Sheet Name | Link to CSV | Description | -| ------------- | ----------------------------- | --------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | -| :green_heart: | case_time_series | | India level timeseries for Confirmed, Recovered and Deceased cases -| :green_heart: | states | | Statewise timeseries of Confirmed, Recovered and Deceased numbers. -| :green_heart: | districts | | Districtwise timeseries of Confirmed, Recovered and Deceased numbers. | -| :green_heart: | state_wise_daily | | Statewise per day delta of Confirmed, Recovered and Deceased numbers. -| :green_heart: | state_wise | | Statewise cumulative numbers till date. | -| :green_heart: | district_wise | | Districtwise Cumulative numbers till date. | -| :green_heart: | statewise_tested_numbers_data | | Number of tests conducted in each state, ventilators ,hospital bed occupany reported in state bulletins | -| :green_heart: | tested_numbers_icmr_data | | Number of tests reported by ICMR | -| :green_heart: | icmr_labs_statewise | | Number of Labs in each state as per ICMR | -| :green_heart: | sources_list | | List of sources that we are using. | -| :green_heart: | rtpcr_samples_collected | | Number of RTPCR samples collected statewise in ICMR Application | -| :green_heart: | vaccine_doses_administered_statewise | | Number of vaccine doses administered statewise - Collected from MOHFW daily bulletin | -| :green_heart: | cowin_vaccine_data_statewise | | Key data points from CoWin database at a state level | -| :green_heart: | cowin_vaccine_data_districtwise | | Key data points from CoWin database at a district level | +| | Sheet Name | Link to CSV | Description | +| --- | ----------------------------- | --------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | +| :green_heart: | case_time_series | | India level timeseries for Confirmed, Recovered and Deceased cases +| :green_heart: | states | | Statewise timeseries of Confirmed, Recovered and Deceased numbers. +| :green_heart: | districts | | Districtwise timeseries of Confirmed, Recovered and Deceased numbers. | +| :green_heart: | state_wise_daily | | Statewise per day delta of Confirmed, Recovered and Deceased numbers. +| :green_heart: | state_wise | | Statewise cumulative numbers till date. | +| :green_heart: | district_wise | | Districtwise Cumulative numbers till date. | +| :green_heart: | statewise_tested_numbers_data | | Number of tests conducted in each state, ventilators ,hospital bed occupany reported in state bulletins | +| :green_heart: | tested_numbers_icmr_data | | Number of tests reported by ICMR | +| :green_heart: | icmr_labs_statewise | | Number of Labs in each state as per ICMR | +| :green_heart: | sources_list | | List of sources that we are using. | +| :green_heart: | rtpcr_samples_collected | | Number of RTPCR samples collected statewise in ICMR Application | +| :green_heart: | vaccine_doses_administered_statewise | | Number of vaccine doses administered statewise - Collected from MOHFW daily bulletin | +| :green_heart: | cowin_vaccine_data_statewise | | Key data points from CoWin database at a state level | +| :green_heart: | cowin_vaccine_data_districtwise | | Key data points from CoWin database at a district level | Latest data from the google sheet (10-20 minutes delayed) is available through the `latest` end-point. These endpoints should be avoided unless none of the above endpoints work for you. @@ -48,41 +47,41 @@ These endpoints should be avoided unless none of the above endpoints work for yo | Status | Sheet Name | Link to CSV | Description | | ------------- | ---------- | -------------------------------------------------------- | ---------------------- | -| :green_heart: | raw_data1 | | Till Apr 19th | -| :green_heart: | raw_data2 | | Apr 20th to Apr 26th | -| :green_heart: | raw_data3 | | Apr 27th to May 9th | -| :green_heart: | raw_data4 | | May 10th to May 23rd | -| :green_heart: | raw_data5 | | May 24th to Jun 4th | -| :green_heart: | raw_data6 | | Jun 05th to Jun 19th | -| :green_heart: | raw_data7 | | Jun 20th to Jun 30th | -| :green_heart: | raw_data8 | | Jul 01st to Jul 7th | -| :green_heart: | raw_data9 | | Jul 08th to Jul 13th | -| :green_heart: | raw_data10 | | Jul 14th to Jul 17th | -| :green_heart: | raw_data11 | | Jul 18th to Jul 22nd | -| :green_heart: | raw_data12 | | Jul 23th to Aug 06th | -| :green_heart: | raw_data13 | | Aug 07th to Aug 21st | -| :green_heart: | raw_data14 | | Aug 22nd to Sep 05th | -| :green_heart: | raw_data15 | | Sep 06th to Sep 21st | -| :green_heart: | raw_data16 | | Sep 22nd to Oct 08th | -| :green_heart: | raw_data17 | | Oct 09th to Oct 26th | -| :green_heart: | raw_data18 | | Oct 27th to Nov 12th | -| :green_heart: | raw_data19 | | Nov 13th to Nov 30th | -| :green_heart: | raw_data20 | | Dec 01st to Dec 19th | -| :green_heart: | raw_data21 | | Dec 20th to Jan 08th | -| :green_heart: | raw_data22 | | Jan 09th to Jan 31st | -| :green_heart: | raw_data23 | | Feb 01st to Feb 27st | -| :green_heart: | raw_data24 | | Feb 28th to Mar 31st | -| :green_heart: | raw_data25 | | Apr 01st to Apr 20th | -| :green_heart: | raw_data26 | | Apr 21st to May 04th | -| :green_heart: | raw_data27 | | May 05th to May 17th | -| :green_heart: | raw_data28 | | May 18th to Jun 02nd | -| :green_heart: | raw_data29 | | Jun 03rd to Jun 19th | -| :green_heart: | raw_data30 | | Jun 20th to Jul 06th | -| :green_heart: | raw_data31 | | Jul 07th to Jul 27th | -| :green_heart: | raw_data32 | | Jul 28th to Aug 07th | -| :green_heart: | raw_data33 | | Aug 08th to Sep 12th | -| :green_heart: | raw_data34 | | Sep 13th to Oct 09th | -| :green_heart: | raw_data35 | | Oct 10th onwards | +| :green_heart: | raw_data1 | | Till Apr 19th | +| :green_heart: | raw_data2 | | Apr 20th to Apr 26th | +| :green_heart: | raw_data3 | | Apr 27th to May 9th | +| :green_heart: | raw_data4 | | May 10th to May 23rd | +| :green_heart: | raw_data5 | | May 24th to Jun 4th | +| :green_heart: | raw_data6 | | Jun 05th to Jun 19th | +| :green_heart: | raw_data7 | | Jun 20th to Jun 30th | +| :green_heart: | raw_data8 | | Jul 01st to Jul 7th | +| :green_heart: | raw_data9 | | Jul 08th to Jul 13th | +| :green_heart: | raw_data10 | | Jul 14th to Jul 17th | +| :green_heart: | raw_data11 | | Jul 18th to Jul 22nd | +| :green_heart: | raw_data12 | | Jul 23th to Aug 06th | +| :green_heart: | raw_data13 | | Aug 07th to Aug 21st | +| :green_heart: | raw_data14 | | Aug 22nd to Sep 05th | +| :green_heart: | raw_data15 | | Sep 06th to Sep 21st | +| :green_heart: | raw_data16 | | Sep 22nd to Oct 08th | +| :green_heart: | raw_data17 | | Oct 09th to Oct 26th | +| :green_heart: | raw_data18 | | Oct 27th to Nov 12th | +| :green_heart: | raw_data19 | | Nov 13th to Nov 30th | +| :green_heart: | raw_data20 | | Dec 01st to Dec 19th | +| :green_heart: | raw_data21 | | Dec 20th to Jan 08th | +| :green_heart: | raw_data22 | | Jan 09th to Jan 31st | +| :green_heart: | raw_data23 | | Feb 01st to Feb 27st | +| :green_heart: | raw_data24 | | Feb 28th to Mar 31st | +| :green_heart: | raw_data25 | | Apr 01st to Apr 20th | +| :green_heart: | raw_data26 | | Apr 21st to May 04th | +| :green_heart: | raw_data27 | | May 05th to May 17th | +| :green_heart: | raw_data28 | | May 18th to Jun 02nd | +| :green_heart: | raw_data29 | | Jun 03rd to Jun 19th | +| :green_heart: | raw_data30 | | Jun 20th to Jul 06th | +| :green_heart: | raw_data31 | | Jul 07th to Jul 27th | +| :green_heart: | raw_data32 | | Jul 28th to Aug 07th | +| :green_heart: | raw_data33 | | Aug 08th to Sep 12th | +| :green_heart: | raw_data34 | | Sep 13th to Oct 09th | +| :green_heart: | raw_data35 | | Oct 10th onwards | #### Note @@ -97,7 +96,7 @@ These endpoints should be avoided unless none of the above endpoints work for yo A more detailed note of the columns present in the data may be found in the json documentation -- [Documentation](https://api.covid19india.org/documentation) +- [Documentation](https://data.covid19bharat.org/documentation) .................................................... diff --git a/log4bash.sh b/log4bash.sh new file mode 100755 index 0000000000..8c17690a33 --- /dev/null +++ b/log4bash.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +#-------------------------------------------------------------------------------------------------- +# log4bash - Makes logging in Bash scripting suck less +# Copyright (c) Fred Palmer +# Licensed under the MIT license +# http://github.com/fredpalmer/log4bash +#-------------------------------------------------------------------------------------------------- +set -e # Fail on first error + +# Useful global variables that users may wish to reference +SCRIPT_ARGS="$@" +SCRIPT_NAME="$0" +SCRIPT_NAME="${SCRIPT_NAME#\./}" +SCRIPT_NAME="${SCRIPT_NAME##/*/}" +SCRIPT_BASE_DIR="$(cd "$( dirname "$0")" && pwd )" + +# This should probably be the right way - didn't have time to experiment though +# declare -r INTERACTIVE_MODE="$([ tty --silent ] && echo on || echo off)" +declare -r INTERACTIVE_MODE=$([ "$(uname)" == "Darwin" ] && echo "on" || echo "off") + +#-------------------------------------------------------------------------------------------------- +# Begin Help Section + +HELP_TEXT="" + +# This function is called in the event of an error. +# Scripts which source this script may override by defining their own "usage" function +usage() { + echo -e "${HELP_TEXT}"; + exit 1; +} + +# End Help Section +#-------------------------------------------------------------------------------------------------- + +#-------------------------------------------------------------------------------------------------- +# Begin Logging Section +if [[ "${INTERACTIVE_MODE}" == "off" ]] +then + # Then we don't care about log colors + declare -r LOG_DEFAULT_COLOR="" + declare -r LOG_ERROR_COLOR="" + declare -r LOG_INFO_COLOR="" + declare -r LOG_SUCCESS_COLOR="" + declare -r LOG_WARN_COLOR="" + declare -r LOG_DEBUG_COLOR="" +else + declare -r LOG_DEFAULT_COLOR="\033[0m" + declare -r LOG_ERROR_COLOR="\033[1;31m" + declare -r LOG_INFO_COLOR="\033[1m" + declare -r LOG_SUCCESS_COLOR="\033[1;32m" + declare -r LOG_WARN_COLOR="\033[1;33m" + declare -r LOG_DEBUG_COLOR="\033[1;34m" +fi + +# This function scrubs the output of any control characters used in colorized output +# It's designed to be piped through with text that needs scrubbing. The scrubbed +# text will come out the other side! +prepare_log_for_nonterminal() { + # Essentially this strips all the control characters for log colors + sed "s/[[:cntrl:]]\[[0-9;]*m//g" +} + +log() { + local log_text="$1" + local log_level="$2" + local log_color="$3" + + # Default level to "info" + [[ -z ${log_level} ]] && log_level="INFO"; + [[ -z ${log_color} ]] && log_color="${LOG_INFO_COLOR}"; + + echo -e "${log_color}[$(date +"%Y-%m-%d %H:%M:%S %Z")] [${log_level}] ${log_text} ${LOG_DEFAULT_COLOR}"; + return 0; +} + +log_info() { log "$@"; } + +log_speak() { + if type -P say >/dev/null + then + local easier_to_say="$1"; + case "${easier_to_say}" in + studionowdev*) + easier_to_say="studio now dev ${easier_to_say#studionowdev}"; + ;; + studionow*) + easier_to_say="studio now ${easier_to_say#studionow}"; + ;; + esac + say "${easier_to_say}"; + fi + return 0; +} + +log_success() { log "$1" "SUCCESS" "${LOG_SUCCESS_COLOR}"; } +log_error() { log "$1" "ERROR" "${LOG_ERROR_COLOR}"; log_speak "$1"; } +log_warning() { log "$1" "WARNING" "${LOG_WARN_COLOR}"; } +log_debug() { log "$1" "DEBUG" "${LOG_DEBUG_COLOR}"; } +log_captains() { + if type -P figlet >/dev/null; + then + figlet -f computer -w 120 "$1"; + else + log "$1"; + fi + + log_speak "$1"; + + return 0; +} + +log_campfire() { + # This function performs a campfire notification with the arguments passed to it + if [[ -z ${CAMPFIRE_API_AUTH_TOKEN} || -z ${CAMPFIRE_NOTIFICATION_URL} ]] + then + log_warning "CAMPFIRE_API_AUTH_TOKEN and CAMPFIRE_NOTIFICATION_URL must be set in order log to campfire." + return 1; + fi + + local campfire_message=" + { + \"message\": { + \"type\":\"TextMessage\", + \"body\":\"$@\" + } + }" + + curl \ + --write-out "\r\n" \ + --user ${CAMPFIRE_API_AUTH_TOKEN}:X \ + --header 'Content-Type: application/json' \ + --data "${campfire_message}" \ + ${CAMPFIRE_NOTIFICATION_URL} + + return $?; +} + +# End Logging Section +#-------------------------------------------------------------------------------------------------- + diff --git a/main.sh b/main.sh old mode 100644 new mode 100755 index 41cef24376..d64ad99fd6 --- a/main.sh +++ b/main.sh @@ -1,61 +1,141 @@ -#!/bin/bash -echo "main.sh start" -set -eu +''' +Main script is responsible for generating all the api related data from various source files +such as google sheets and other sources. It eventually creates two set of data files including +minified version of the data -repo_uri="https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" -remote_name="origin" -main_branch="main" -gh_pages_branch="gh-pages" +Bash script utilises both node.js scripts and python scripts for certain functionalities +Prerequisites: +main.sh requires executable permissions so ensure you have set the executable permission 755 +using the below command +sudo chmod 755 main.sh -git config user.name "$GITHUB_ACTOR" -git config user.email "${GITHUB_ACTOR}@bots.github.com" +In order to commit the data back to the github repo, you will need a personal access token +that can be generated from github.com/settings/tokens +make a copy of the env.example file and add your personal token in the environment variables to be +used across the project and renamed the file to .env +''' +#!/usr/bin/env bash +source log4bash.sh -git checkout "$gh_pages_branch" +# Set the environment variables from the .env file using setenv.sh script +source setenv.sh +set -eu -mkdir tmp +# Set to dev branch by default +active_branch=dev -cp -r ./updatelog ./tmp -cp -r ./csv ./tmp -cp v4/min/data.min.json ./tmp/data-old.min.json -cp ./csv/latest/state_wise.csv ./tmp/state_wise_prev +if [ "${MODE}" == 'prod' ]; then + echo "Production mode: Be careful with this mode." + active_branch=${MAIN_BRANCH} +else + echo "Developer mode: You can play around." + active_branch=${DEV_BRANCH} +fi +# Setting the repo path and branche +repo_uri="https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" -git checkout "$main_branch" +# Download all necessary files from repo branches in to code directory +if [ -d "${CODE_DIR}" ]; then + echo "${CODE_DIR} directory exists" + cd ${CODE_DIR} +else + echo "Creating new directory named ${CODE_DIR}..." + mkdir -p ${CODE_DIR} && cd $_ +fi +# Chekout repo branches in respective folders +if [ -d "${GH_PAGES_BRANCH}" ]; +then + cd ${GH_PAGES_BRANCH} + git checkout ${GH_PAGES_BRANCH} + cd .. +else + git clone --depth 1 -b ${GH_PAGES_BRANCH} $repo_uri ${GH_PAGES_BRANCH} +fi -cp README.md tmp/ -cp -r documentation/ tmp/ -node src/sheets-to-csv.js +if [ -d "${active_branch}" ]; +then + cd $active_branch + git checkout $active_branch + cd .. +else + git clone --depth 1 -b $active_branch $repo_uri $active_branch +fi -python3 src/parser_v4.py -python3 src/generate_activity_log.py -# node src/sanity_check.js # need rewrite with new json +if [ -d "${TEMP_DIR}" ]; then + echo "${TEMP_DIR} directory exists" +else + echo "Creating new directory named ${TEMP_DIR}..." + mkdir ${TEMP_DIR} +fi + +# Verify the existence of the directory +if [ -d "${GH_PAGES_BRANCH}" ]; then + echo "${GH_PAGES_BRANCH} directory exists" + # Copying files to respective folders + cp -r ${GH_PAGES_BRANCH}/updatelog ${TEMP_DIR} + cp -r ${GH_PAGES_BRANCH}/csv ${TEMP_DIR} + cp ${GH_PAGES_BRANCH}/v4/min/data.min.json ${TEMP_DIR}/data-old.min.json + cp ${GH_PAGES_BRANCH}/csv/latest/state_wise.csv ${TEMP_DIR}/state_wise_prev +else + echo "${GH_PAGES_BRANCH} not found. Exiting..." +fi + +# Verify the existence of the directory +if [ -d "$active_branch" ]; then + echo "$active_branch directory exists" + # Copying files to respective folders + cp ./$active_branch/README.md ${TEMP_DIR}/ + cp -r ./$active_branch/documentation/ ${TEMP_DIR}/ +else + echo "$active_branch not found. Exiting..." +fi +# Convert the google sheet data to csv using Node.js script +node ./$active_branch/src/sheets-to-csv.js -git checkout "$gh_pages_branch" +# Invoke the Python Parser 4 script to generate the json data for api calls +python3 ./$active_branch/src/parser_v4.py +python3 ./$active_branch/src/generate_activity_log.py +# node src/sanity_check.js # need rewrite with new json -rm tmp/data-old.min.json -rm tmp/state_wise_prev +# Remove the old files from tmp directory +rm ${TEMP_DIR}/data-old.min.json +rm ${TEMP_DIR}/state_wise_prev -cp -r tmp/* . -rm -r tmp/ +# # Copy everything from tmp directory to root folder and remove tmp directory +cp -r ${TEMP_DIR}/* ${GH_PAGES_BRANCH} +rm -r ${TEMP_DIR} +cd ${GH_PAGES_BRANCH} -git add . -set +e # Grep succeeds with nonzero exit codes to show results. +if [ "${MODE}" == 'prod' ]; then + # Housekeeping + git config user.name "$GITHUB_ACTOR" + git config user.email "${GITHUB_ACTOR}@bots.github.com" + # Add all the files to the repo and commit + git add . + set +e # Grep succeeds with nonzero exit codes to show results. +else + git config --global user.name "${GITHUB_NAME}" + git config --global user.email "${GITHUB_ACTOR}" + echo "Dev mode: So not committing..." +fi +# Commit the changes if there are new modifications or files. if git status | grep 'new file\|modified' then - set -e - git commit -am "data updated on - $(date)" - git remote set-url "$remote_name" "$repo_uri" # includes access token - git push --force-with-lease "$remote_name" "$gh_pages_branch" -else - set -e - echo "No changes since last run" + echo "Prod mode: Committing all changes" + git commit -am "data updated on - $(date)" + git remote set-url "${ORIGIN_BRANCH}" "$repo_uri" # includes access token + git push --force-with-lease "${ORIGIN_BRANCH}" "${GH_PAGES_BRANCH}" + rm -rf ../../${CODE_DIR}/ fi -echo "main.sh end" \ No newline at end of file + + +echo "Script successfully completed" diff --git a/package-lock.json b/package-lock.json deleted file mode 100644 index 1c5d755ee5..0000000000 --- a/package-lock.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "name": "api", - "version": "1.0.0", - "lockfileVersion": 1, - "requires": true, - "dependencies": { - "date-fns": { - "version": "2.13.0", - "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.13.0.tgz", - "integrity": "sha512-xm0c61mevGF7f0XpCGtDTGpzEFC/1fpLXHbmFpxZZQJuvByIK2ozm6cSYuU+nxFYOPh2EuCfzUwlTEFwKG+h5w==" - }, - "drive-db": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/drive-db/-/drive-db-6.0.0.tgz", - "integrity": "sha512-DkMtcLcL0O4FCYMLgecXoeNwRKLprCS5dgD2FkXPylt/EakbzPoMoDJSO3qGuN9MqUN22mdF67YOKfcLtZrHLQ==" - }, - "immer": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/immer/-/immer-8.0.1.tgz", - "integrity": "sha512-aqXhGP7//Gui2+UrEtvxZxSquQVXTpZ7KDxfCcKAF3Vysvw0CViVaW9RZ1j1xlIYqaaaipBoqdqeibkc18PNvA==" - }, - "moment": { - "version": "2.24.0", - "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", - "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==" - }, - "moment-timezone": { - "version": "0.5.28", - "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.28.tgz", - "integrity": "sha512-TDJkZvAyKIVWg5EtVqRzU97w0Rb0YVbfpqyjgu6GwXCAohVRqwZjf4fOzDE6p1Ch98Sro/8hQQi65WDXW5STPw==", - "requires": { - "moment": ">= 2.9.0" - } - }, - "node-fetch": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", - "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" - } - } -} diff --git a/setenv.sh b/setenv.sh new file mode 100644 index 0000000000..fa5fc26e78 --- /dev/null +++ b/setenv.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +# Show env vars +grep -v '^#' .env + +# Export env vars +export $(grep -v '^#' .env | xargs) diff --git a/src/parser_v4.py b/src/parser_v4.py index 5a1e12896c..e3d411e39a 100755 --- a/src/parser_v4.py +++ b/src/parser_v4.py @@ -8,6 +8,15 @@ from collections import defaultdict, OrderedDict from datetime import datetime, timedelta from pathlib import Path +import os +from dotenv import load_dotenv +import traceback +from pathlib import Path + +load_dotenv() + +ROOT_DIR = Path(__file__).parents[2] +print(ROOT_DIR) # Set logging level logging.basicConfig(stream=sys.stdout, @@ -24,8 +33,8 @@ MIN_DATE = "2020-01-01" # Input/Output root directory -ROOT_DIR = Path("tmp") -CSV_DIR = ROOT_DIR / "csv" / "latest" +# ROOT_DIR = Path("tmp") +CSV_DIR = ROOT_DIR / "tmp" / "csv" / "latest" # State codes to be used as API keys STATE_META_DATA = CSV_DIR / "states_meta.csv" DISTRICT_META_DATA = CSV_DIR / "districts_meta.csv" @@ -45,7 +54,8 @@ STATE_VACCINATION_DATA = CSV_DIR / "vaccine_doses_statewise_v2.csv" DISTRICT_VACCINATION_DATA = CSV_DIR / "cowin_vaccine_data_districtwise.csv" # Old data.json -DATA_OLD = ROOT_DIR / "data-old.min.json" +DATA_OLD = ROOT_DIR / "tmp" / "data-old.min.json" +print(DATA_OLD) ## For adding metadata # For state notes and last updated @@ -54,7 +64,7 @@ DISTRICT_WISE = CSV_DIR / "district_wise.csv" # API outputs -OUTPUT_DIR = ROOT_DIR / "v4" +OUTPUT_DIR = ROOT_DIR / "tmp" / "v4" OUTPUT_MIN_DIR = OUTPUT_DIR / "min" OUTPUT_DATA_PREFIX = "data" OUTPUT_TIMESERIES_PREFIX = "timeseries" @@ -101,10 +111,16 @@ "key": "Second Dose Administered", "source": "Source 4" }, + "precautionary": { + "key": "Precautionary Dose Administered", + "source": "Source 4" + }, } VACCINATION_DATA_DICT = { "vaccinated1": "First Dose Administered", "vaccinated2": "Second Dose Administered", + "precautionary": "Precautionary Dose Administered", + } ALL_STATISTICS = [*RAW_DATA_MAP.values(), *ICMR_DATA_DICT.keys()] # CSV Headers @@ -369,40 +385,41 @@ def parse_icmr(reader): for j, entry in enumerate(reader): for statistic, statistic_dict in ICMR_DATA_DICT.items(): key = statistic_dict["key"] - count_str = entry[key].strip() + if key in entry: + count_str = entry[key].strip() - if not count_str: - continue + if not count_str: + continue - try: - fdate = datetime.strptime(entry["Tested As Of"].strip(), "%d/%m/%Y") - date = datetime.strftime(fdate, "%Y-%m-%d") - if date < MIN_DATE or date > INDIA_DATE: - # Entries from future dates will be ignored and logged - logging.warning( - f"[L{j + 2}] [Future/past date: {entry['Tested As Of']}]") + try: + fdate = datetime.strptime(entry["Tested As Of"].strip(), "%d/%m/%Y") + date = datetime.strftime(fdate, "%Y-%m-%d") + if date < MIN_DATE or date > INDIA_DATE: + # Entries from future dates will be ignored and logged + logging.warning( + f"[L{j + 2}] [Future/past date: {entry['Tested As Of']}]") + continue + except ValueError: + # Bad timestamp + logging.warning(f"[L{j + 2}] [Bad date: {entry['Tested As Of']}]") continue - except ValueError: - # Bad timestamp - logging.warning(f"[L{j + 2}] [Bad date: {entry['Tested As Of']}]") - continue - try: - count = int(count_str) - except ValueError: - logging.warning( - f"[L{j + 2}] [{entry['Tested As Of']}] [Bad {key}: {entry[key]}]") - continue + try: + count = int(count_str) + except ValueError: + logging.warning( + f"[L{j + 2}] [{entry['Tested As Of']}] [Bad {key}: {entry[key]}]") + continue - if count: - data[date]["TT"]["total"][statistic] = count + if count: + data[date]["TT"]["total"][statistic] = count - # Add source/last updated - meta_key = ("vaccinated" if statistic - in {"vaccinated1", "vaccinated2"} else statistic) - data[date]["TT"]["meta"][meta_key]["source"] = entry[ - statistic_dict["source"]].strip() - data[date]["TT"]["meta"][meta_key]["date"] = date + # Add source/last updated + meta_key = ("vaccinated" if statistic + in {"vaccinated1", "vaccinated2"} else statistic) + data[date]["TT"]["meta"][meta_key]["source"] = entry[ + statistic_dict["source"]].strip() + data[date]["TT"]["meta"][meta_key]["date"] = date def parse_state_test(reader): @@ -648,22 +665,25 @@ def parse_district_vaccination(reader): for statistic in VACCINATION_DATA_DICT: key = VACCINATION_DATA_DICT[statistic].lower() - count_str = row[j2 + column_keys[key]].strip() - try: - count = int(count_str) - except ValueError: - if count_str: - logging.warning( - f"[L{i + 3} {column_str(j2 + column_keys[key] + 1)}] [{state}: {district}] Bad {key}: {row[j2 + column_keys[key]]}" - ) - continue - if count: - inc( - data[date][state]["districts"][district]["total"], - statistic, - count, - ) + # To handle the empty precautionary column in the data before 15-Jan-2022 + if key in column_keys: + count_str = row[j2 + column_keys[key]].strip() + try: + count = int(count_str) + except ValueError: + if count_str: + logging.warning( + f"[L{i + 3} {column_str(j2 + column_keys[key] + 1)}] [{state}: {district}] Bad {key}: {row[j2 + column_keys[key]]}" + ) + continue + + if count: + inc( + data[date][state]["districts"][district]["total"], + statistic, + count, + ) def contains(raw_data, keys): @@ -1152,281 +1172,287 @@ def write_csvs(writer_states, writer_districts): if __name__ == "__main__": - logging.info("-" * PRINT_WIDTH) - logging.info("PARSER V4 START".center(PRINT_WIDTH)) - - # Get possible state codes, populations - logging.info("-" * PRINT_WIDTH) - logging.info("Parsing state metadata...") - with open(STATE_META_DATA) as f: - logging.info(f"File: {STATE_META_DATA.name}") - reader = csv.DictReader(f) - parse_state_metadata(reader) - logging.info("Done!") - # Get all actual district names - logging.info("-" * PRINT_WIDTH) - logging.info("Parsing districts list...") - with open(DISTRICT_LIST) as f: - logging.info(f"File: {DISTRICT_LIST.name}") - reader = csv.DictReader(f) - parse_district_list(reader) - logging.info("Done!") - - # Get district populations - logging.info("-" * PRINT_WIDTH) - logging.info("Parsing district metadata...") - with open(DISTRICT_META_DATA) as f: - logging.info(f"File: {DISTRICT_META_DATA.name}") - reader = csv.DictReader(f) - parse_district_metadata(reader) - logging.info("Done!") + try: + logging.info("-" * PRINT_WIDTH) + logging.info("PARSER V4 START".center(PRINT_WIDTH)) - # Parse raw_data's - logging.info("-" * PRINT_WIDTH) - logging.info("Parsing raw_data...") - i = 1 - while True: - fn = CSV_DIR / RAW_DATA.format(n=i) - if not fn.is_file(): - break - with open(fn) as f: - logging.info(f"File: {fn.name}") + # Get possible state codes, populations + logging.info("-" * PRINT_WIDTH) + logging.info("Parsing state metadata...") + with open(STATE_META_DATA) as f: + logging.info(f"File: {STATE_META_DATA.name}") reader = csv.DictReader(f) - parse_raw_data(reader, i) - i += 1 - logging.info("Done!") - - # Parse additional deceased/recovered info not in raw_data 1 and 2 - logging.info("-" * PRINT_WIDTH) - logging.info("Parsing deaths_recoveries...") - for i in [1, 2]: - fn = CSV_DIR / OUTCOME_DATA.format(n=i) - with open(fn) as f: - logging.info(f"File: {fn.name}") + parse_state_metadata(reader) + logging.info("Done!") + + # Get all actual district names + logging.info("-" * PRINT_WIDTH) + logging.info("Parsing districts list...") + with open(DISTRICT_LIST) as f: + logging.info(f"File: {DISTRICT_LIST.name}") reader = csv.DictReader(f) - parse_outcome(reader, i) - logging.info("Done!") - - logging.info("-" * PRINT_WIDTH) - logging.info("Adding district data for 26th April...") - # Parse gospel district data for 26th April - with open(DISTRICT_DATA_GOSPEL) as f: - logging.info(f"File: {DISTRICT_DATA_GOSPEL.name}") - reader = csv.DictReader(f) - parse_district_gospel(reader) - logging.info("Done!") - - logging.info("-" * PRINT_WIDTH) - logging.info("Parsing ICMR test data for India...") - with open(ICMR_TEST_DATA) as f: - logging.info(f"File: {ICMR_TEST_DATA.name}") - reader = csv.DictReader(f) - parse_icmr(reader) - logging.info("Done!") - - logging.info("-" * PRINT_WIDTH) - logging.info("Parsing test data for all states...") - with open(STATE_TEST_DATA) as f: - logging.info(f"File: {STATE_TEST_DATA.name}") - reader = csv.DictReader(f) - parse_state_test(reader) - logging.info("Done!") - - logging.info("-" * PRINT_WIDTH) - logging.info("Parsing test data for districts...") - with open(DISTRICT_TEST_DATA) as f: - logging.info(f"File: {DISTRICT_TEST_DATA.name}") - reader = csv.reader(f) - parse_district_test(reader) - logging.info("Done!") - - logging.info("-" * PRINT_WIDTH) - logging.info("Parsing vaccination data for states...") - with open(STATE_VACCINATION_DATA) as f: - logging.info(f"File: {STATE_VACCINATION_DATA.name}") - reader = csv.DictReader(f) - parse_state_vaccination(reader) - logging.info("Done!") - - logging.info("-" * PRINT_WIDTH) - logging.info("Parsing vaccination data for districts...") - with open(DISTRICT_VACCINATION_DATA) as f: - logging.info(f"File: {DISTRICT_VACCINATION_DATA.name}") - reader = csv.reader(f) - parse_district_vaccination(reader) - logging.info("Done!") - - # Fill delta values for tested - logging.info("-" * PRINT_WIDTH) - logging.info("Generating daily tested/vaccinated values...") - fill_deltas() - logging.info("Done!") - - # Generate total (cumulative) data points till 26th April - logging.info("-" * PRINT_WIDTH) - logging.info("Generating cumulative CRD values till 26th April...") - accumulate(end_date=GOSPEL_DATE) - logging.info("Done!") - - # Fill Unknown district counts for 26th April - logging.info("-" * PRINT_WIDTH) - logging.info(f"Filling {UNKNOWN_DISTRICT_KEY} data for 26th April...") - fill_gospel_unknown() - logging.info("Done!") - - # Generate rest of total (cumulative) data points - logging.info("-" * PRINT_WIDTH) - logging.info( - "Generating cumulative CRD values from 26th April afterwards...") - accumulate(start_after_date=GOSPEL_DATE) - logging.info("Done!") - - # Generate 7 day delta values - logging.info("-" * PRINT_WIDTH) - logging.info("Generating 7-day delta values...") - accumulate_days(7) - logging.info("Done!") - - # Generate 14-21 day confirmed delta values - logging.info("-" * PRINT_WIDTH) - logging.info("Generating 14-21 day confirmed delta values...") - accumulate_days(21, offset=14, statistics=["confirmed"]) - logging.info("Done!") - - # Strip empty values ({}, 0, '', None) before adding metadata - logging.info("-" * PRINT_WIDTH) - logging.info("Stripping empty values...") - data = stripper(data) - logging.info("Done!") - - # Add population figures - logging.info("-" * PRINT_WIDTH) - logging.info("Adding state/district populations...") - add_populations() - logging.info("Done!") - - # Add state notes - logging.info("-" * PRINT_WIDTH) - logging.info("Adding state and district notes...") - with open(STATE_WISE) as f: - logging.info(f"File: {STATE_WISE.name}") - reader = csv.DictReader(f) - add_state_notes(reader) - - # Add district notes - with open(DISTRICT_WISE) as f: - logging.info(f"File: {DISTRICT_WISE.name}") - reader = csv.DictReader(f) - add_district_notes(reader) - logging.info("Done!") - - # Add last updated time for states - logging.info("-" * PRINT_WIDTH) - logging.info("Adding last updated time for states...") - with open(DATA_OLD) as f: - logging.info(f"File: {DATA_OLD.name}") - data_old = json.load(f, ) - add_state_last_updated(data_old) - logging.info("Done!") - - # Generate timeseries - logging.info("-" * PRINT_WIDTH) - logging.info("Generating timeseries...") - generate_timeseries(districts=True) - logging.info("Done!") + parse_district_list(reader) + logging.info("Done!") + + # Get district populations + logging.info("-" * PRINT_WIDTH) + logging.info("Parsing district metadata...") + with open(DISTRICT_META_DATA) as f: + logging.info(f"File: {DISTRICT_META_DATA.name}") + reader = csv.DictReader(f) + parse_district_metadata(reader) + logging.info("Done!") + + # Parse raw_data's + logging.info("-" * PRINT_WIDTH) + logging.info("Parsing raw_data...") + i = 1 + while True: + fn = CSV_DIR / RAW_DATA.format(n=i) + if not fn.is_file(): + break + with open(fn) as f: + logging.info(f"File: {fn.name}") + reader = csv.DictReader(f) + parse_raw_data(reader, i) + i += 1 + logging.info("Done!") + + # Parse additional deceased/recovered info not in raw_data 1 and 2 + logging.info("-" * PRINT_WIDTH) + logging.info("Parsing deaths_recoveries...") + for i in [1, 2]: + fn = CSV_DIR / OUTCOME_DATA.format(n=i) + with open(fn) as f: + logging.info(f"File: {fn.name}") + reader = csv.DictReader(f) + parse_outcome(reader, i) + logging.info("Done!") + + logging.info("-" * PRINT_WIDTH) + logging.info("Adding district data for 26th April...") + # Parse gospel district data for 26th April + with open(DISTRICT_DATA_GOSPEL) as f: + logging.info(f"File: {DISTRICT_DATA_GOSPEL.name}") + reader = csv.DictReader(f) + parse_district_gospel(reader) + logging.info("Done!") - logging.info("-" * PRINT_WIDTH) - logging.info("Dumping JSON APIs...") - OUTPUT_MIN_DIR.mkdir(parents=True, exist_ok=True) - - # Dump prettified full data json - # fn = f"{OUTPUT_DATA_PREFIX}-all" - # Only dump minified data-all.json - # with open((OUTPUT_DIR / fn).with_suffix('.json'), 'w') as f: - # json.dump(data, f, indent=2, sort_keys=True) - # Dump minified full data - # with open((OUTPUT_MIN_DIR / fn).with_suffix(".min.json"), "w") as f: - # json.dump(data, f, separators=(",", ":"), sort_keys=True) - - # Split data and dump separate json for each date - for i, date in enumerate(sorted(data)): - curr_data = data[date] - if i < len(data) - 1: - fn = f"{OUTPUT_DATA_PREFIX}-{date}" - else: - fn = OUTPUT_DATA_PREFIX + logging.info("-" * PRINT_WIDTH) + logging.info("Parsing ICMR test data for India...") + with open(ICMR_TEST_DATA) as f: + logging.info(f"File: {ICMR_TEST_DATA.name}") + reader = csv.DictReader(f) + parse_icmr(reader) + logging.info("Done!") - with open((OUTPUT_DIR / fn).with_suffix(".json"), "w") as f: - json.dump(curr_data, f, indent=2, sort_keys=True) - # Minified - with open((OUTPUT_MIN_DIR / fn).with_suffix(".min.json"), "w") as f: - json.dump(curr_data, f, separators=(",", ":"), sort_keys=True) - - # Dump full timeseries json - # fn = f"{OUTPUT_TIMESERIES_PREFIX}-all" - # Only dump minified timeseries-all.json - # with open((OUTPUT_DIR / fn).with_suffix('.json'), 'w') as f: - # json.dump(timeseries, f, indent=2, sort_keys=True) - # with open((OUTPUT_MIN_DIR / fn).with_suffix(".min.json"), "w") as f: - # json.dump(timeseries, f, separators=(",", ":"), sort_keys=True) - - # Dump state timeseries json - fn = OUTPUT_TIMESERIES_PREFIX - # Filter out district time-series - timeseries_states = { - state: { - "dates": timeseries[state]["dates"] - } - for state in timeseries - } - with open((OUTPUT_DIR / fn).with_suffix(".json"), "w") as f: - json.dump(timeseries_states, f, indent=2, sort_keys=True) - with open((OUTPUT_MIN_DIR / fn).with_suffix(".min.json"), "w") as f: - json.dump(timeseries_states, f, separators=(",", ":"), sort_keys=True) - - # Split data and dump separate json for each state - for state in timeseries: - if state == UNASSIGNED_STATE_CODE: - continue - state_data = {state: timeseries[state]} - fn = f"{OUTPUT_TIMESERIES_PREFIX}-{state}" + logging.info("-" * PRINT_WIDTH) + logging.info("Parsing test data for all states...") + with open(STATE_TEST_DATA) as f: + logging.info(f"File: {STATE_TEST_DATA.name}") + reader = csv.DictReader(f) + parse_state_test(reader) + logging.info("Done!") + + logging.info("-" * PRINT_WIDTH) + logging.info("Parsing test data for districts...") + with open(DISTRICT_TEST_DATA) as f: + logging.info(f"File: {DISTRICT_TEST_DATA.name}") + reader = csv.reader(f) + parse_district_test(reader) + logging.info("Done!") + + logging.info("-" * PRINT_WIDTH) + logging.info("Parsing vaccination data for states...") + with open(STATE_VACCINATION_DATA) as f: + logging.info(f"File: {STATE_VACCINATION_DATA.name}") + reader = csv.DictReader(f) + parse_state_vaccination(reader) + logging.info("Done!") + + logging.info("-" * PRINT_WIDTH) + logging.info("Parsing vaccination data for districts...") + with open(DISTRICT_VACCINATION_DATA) as f: + logging.info(f"File: {DISTRICT_VACCINATION_DATA.name}") + reader = csv.reader(f) + parse_district_vaccination(reader) + logging.info("Done!") + + # Fill delta values for tested + logging.info("-" * PRINT_WIDTH) + logging.info("Generating daily tested/vaccinated values...") + fill_deltas() + logging.info("Done!") + + # Generate total (cumulative) data points till 26th April + logging.info("-" * PRINT_WIDTH) + logging.info("Generating cumulative CRD values till 26th April...") + accumulate(end_date=GOSPEL_DATE) + logging.info("Done!") + + # Fill Unknown district counts for 26th April + logging.info("-" * PRINT_WIDTH) + logging.info(f"Filling {UNKNOWN_DISTRICT_KEY} data for 26th April...") + fill_gospel_unknown() + logging.info("Done!") + + # Generate rest of total (cumulative) data points + logging.info("-" * PRINT_WIDTH) + logging.info( + "Generating cumulative CRD values from 26th April afterwards...") + accumulate(start_after_date=GOSPEL_DATE) + logging.info("Done!") + + # Generate 7 day delta values + logging.info("-" * PRINT_WIDTH) + logging.info("Generating 7-day delta values...") + accumulate_days(7) + logging.info("Done!") + + # Generate 14-21 day confirmed delta values + logging.info("-" * PRINT_WIDTH) + logging.info("Generating 14-21 day confirmed delta values...") + accumulate_days(21, offset=14, statistics=["confirmed"]) + logging.info("Done!") + + # Strip empty values ({}, 0, '', None) before adding metadata + logging.info("-" * PRINT_WIDTH) + logging.info("Stripping empty values...") + data = stripper(data) + logging.info("Done!") + + # Add population figures + logging.info("-" * PRINT_WIDTH) + logging.info("Adding state/district populations...") + add_populations() + logging.info("Done!") + + # Add state notes + logging.info("-" * PRINT_WIDTH) + logging.info("Adding state and district notes...") + with open(STATE_WISE) as f: + logging.info(f"File: {STATE_WISE.name}") + reader = csv.DictReader(f) + add_state_notes(reader) + # Add district notes + with open(DISTRICT_WISE) as f: + logging.info(f"File: {DISTRICT_WISE.name}") + reader = csv.DictReader(f) + add_district_notes(reader) + logging.info("Done!") + + # Add last updated time for states + logging.info("-" * PRINT_WIDTH) + logging.info("Adding last updated time for states...") + with open(DATA_OLD) as f: + logging.info(f"File: {DATA_OLD.name}") + data_old = json.load(f, ) + add_state_last_updated(data_old) + logging.info("Done!") + + # Generate timeseries + logging.info("-" * PRINT_WIDTH) + logging.info("Generating timeseries...") + generate_timeseries(districts=True) + logging.info("Done!") + + logging.info("-" * PRINT_WIDTH) + logging.info("Dumping JSON APIs...") + OUTPUT_MIN_DIR.mkdir(parents=True, exist_ok=True) + + # Dump prettified full data json + # fn = f"{OUTPUT_DATA_PREFIX}-all" + # Only dump minified data-all.json + # with open((OUTPUT_DIR / fn).with_suffix('.json'), 'w') as f: + # json.dump(data, f, indent=2, sort_keys=True) + # Dump minified full data + # with open((OUTPUT_MIN_DIR / fn).with_suffix(".min.json"), "w") as f: + # json.dump(data, f, separators=(",", ":"), sort_keys=True) + + + # Split data and dump separate json for each date in v4 folder + for i, date in enumerate(sorted(data)): + curr_data = data[date] + if i < len(data) - 1: + fn = f"{OUTPUT_DATA_PREFIX}-{date}" + else: + fn = OUTPUT_DATA_PREFIX + + with open((OUTPUT_DIR / fn).with_suffix(".json"), "w") as f: + json.dump(curr_data, f, indent=2, sort_keys=True) + # Minified + with open((OUTPUT_MIN_DIR / fn).with_suffix(".min.json"), "w") as f: + json.dump(curr_data, f, separators=(",", ":"), sort_keys=True) + + # Dump full timeseries json + # fn = f"{OUTPUT_TIMESERIES_PREFIX}-all" + # Only dump minified timeseries-all.json + # with open((OUTPUT_DIR / fn).with_suffix('.json'), 'w') as f: + # json.dump(timeseries, f, indent=2, sort_keys=True) + # with open((OUTPUT_MIN_DIR / fn).with_suffix(".min.json"), "w") as f: + # json.dump(timeseries, f, separators=(",", ":"), sort_keys=True) + + # Dump state timeseries json + fn = OUTPUT_TIMESERIES_PREFIX + # Filter out district time-series + timeseries_states = { + state: { + "dates": timeseries[state]["dates"] + } + for state in timeseries + } with open((OUTPUT_DIR / fn).with_suffix(".json"), "w") as f: - json.dump(state_data, f, indent=2, sort_keys=True) - # Minified + json.dump(timeseries_states, f, indent=2, sort_keys=True) with open((OUTPUT_MIN_DIR / fn).with_suffix(".min.json"), "w") as f: - json.dump(state_data, f, separators=(",", ":"), sort_keys=True) - logging.info("Done!") - - # Tally final date counts with statewise API - logging.info("-" * PRINT_WIDTH) - logging.info("Comparing data with statewise sheet...") - with open(STATE_WISE) as f: - logging.info(f"File: {STATE_WISE.name}") - reader = csv.DictReader(f) - tally_statewise(reader) - logging.info("Done!") + json.dump(timeseries_states, f, separators=(",", ":"), sort_keys=True) - # Tally final date counts with districtwise API - logging.info("-" * PRINT_WIDTH) - logging.info("Comparing data with districtwise sheet...") - with open(DISTRICT_WISE) as f: - logging.info(f"File: {DISTRICT_WISE.name}") - reader = csv.DictReader(f) - tally_districtwise(reader) - logging.info("Done!") - - # Dump state/district CSVs - logging.info("-" * PRINT_WIDTH) - logging.info("Dumping CSVs...") - with open(OUTPUT_STATES_CSV, "w") as f1: - writer1 = csv.writer(f1) - with open(OUTPUT_DISTRICTS_CSV, "w") as f2: - writer2 = csv.writer(f2) - write_csvs(writer1, writer2) - logging.info("Done!") - - logging.info("-" * PRINT_WIDTH) - logging.info("PARSER V4 END".center(PRINT_WIDTH)) - logging.info("-" * PRINT_WIDTH) + # Split data and dump separate json for each state + for state in timeseries: + if state == UNASSIGNED_STATE_CODE: + continue + state_data = {state: timeseries[state]} + fn = f"{OUTPUT_TIMESERIES_PREFIX}-{state}" + + with open((OUTPUT_DIR / fn).with_suffix(".json"), "w") as f: + json.dump(state_data, f, indent=2, sort_keys=True) + # Minified + with open((OUTPUT_MIN_DIR / fn).with_suffix(".min.json"), "w") as f: + json.dump(state_data, f, separators=(",", ":"), sort_keys=True) + logging.info("Done!") + + # Tally final date counts with statewise API + logging.info("-" * PRINT_WIDTH) + logging.info("Comparing data with statewise sheet...") + with open(STATE_WISE) as f: + logging.info(f"File: {STATE_WISE.name}") + reader = csv.DictReader(f) + tally_statewise(reader) + logging.info("Done!") + + # Tally final date counts with districtwise API + logging.info("-" * PRINT_WIDTH) + logging.info("Comparing data with districtwise sheet...") + with open(DISTRICT_WISE) as f: + logging.info(f"File: {DISTRICT_WISE.name}") + reader = csv.DictReader(f) + tally_districtwise(reader) + logging.info("Done!") + + # Dump state/district CSVs + logging.info("-" * PRINT_WIDTH) + logging.info("Dumping CSVs...") + with open(OUTPUT_STATES_CSV, "w") as f1: + writer1 = csv.writer(f1) + with open(OUTPUT_DISTRICTS_CSV, "w") as f2: + writer2 = csv.writer(f2) + write_csvs(writer1, writer2) + logging.info("Done!") + + logging.info("-" * PRINT_WIDTH) + logging.info("PARSER V4 END".center(PRINT_WIDTH)) + logging.info("-" * PRINT_WIDTH) + + except Exception as err: + traceback.print_tb(err.__traceback__) diff --git a/src/sheets-to-csv.js b/src/sheets-to-csv.js index c9f8916aa1..4dff500add 100644 --- a/src/sheets-to-csv.js +++ b/src/sheets-to-csv.js @@ -13,78 +13,45 @@ if (!fs.existsSync(latestDir)) { } // Published sheets -const PUBLISHED_SHEET_ID_1 = - "2PACX-1vSz8Qs1gE_IYpzlkFkCXGcL_BqR8hZieWVi-rphN1gfrO3H4lDtVZs4kd0C3P8Y9lhsT1rhoB-Q_cP4"; -const PUBLISHED_SHEET_ID_2 = - "2PACX-1vRodtoTyQwXckfuvuQllkMhGC_gruigaaizVc8I6-BZWeetYpmRyexnO75ep7rnSxFICd8c9dfpwU8I"; -const PUBLISHED_SHEET_ID_3 = - "2PACX-1vR_17UovavD4X7m_pqzmXjA_kCjGxIapemdWpRhDELHR1LbLJ-EVbxjKgeQat489BFRZ9bqMf-ILe_H"; -const PUBLISHED_SHEET_ID_4 = - "2PACX-1vSeAoAk_iMv7cQ0tldZC7aivJmGKM5Wpc5VVr37Nzv-geTmtr6pDMb-oDK59RS21Om80-SYR3jRp6qq"; -const PUBLISHED_SHEET_ID_5 = - "2PACX-1vSEikAgjAB9x7yhx4zNOUGLIx8Zfy2mAzRv0K1tbw08g73MO88-bbWCsgmhJ0uXa0gtuUlLMOnE9h26"; -const PUBLISHED_SHEET_ID_6 = - "2PACX-1vQQmgjCktQknnTPy-s4OFycu-imtoMCrWY5M2Lqig3nhGyy6W5E27xbCyaaKV9lGaDWmTzGWVzPH9-S"; -const PUBLISHED_SHEET_ID_7 = - "2PACX-1vR6blqV85tiBO-9u4MCW72qXALS3f7yQD0iV47MbsmIcKrvBDTorIVrUJ96QrxUj7iwAviYiecjp8VU"; -const PUBLISHED_SHEET_ID_8 = - "2PACX-1vR1zl3JStozuCgPsPol19f9k_io1ABmHS_mOl9gzWxiDd2_WvWhdfhePXBFZIUFjpW-gPfPwE9m7AA_"; -const PUBLISHED_SHEET_ID_9 = - "2PACX-1vRb4AsEPrV4b0S4j2vQku-J5XHnh8c_8fzmIhD2S2aMc2if7g6bLwJNYOPV8UmrrNR-Bv0C0yjcUnU3"; -const PUBLISHED_SHEET_ID_10 = - "2PACX-1vQyBRow24Pc7Wm_mSjU3JDy_Ua5mFByz6zE7-vFguBvUOdcr-90PgNcTBOCL-nTa40WrghiAN-kSFVX"; -const PUBLISHED_SHEET_ID_11 = - "2PACX-1vTd_tTI33CBI4obGaKTo0dfw1cNu5dUz4OIIhdbWJVmZJlEVslMyWzky1ifb9uRmV0siVxneBW4iBwi"; -const PUBLISHED_SHEET_ID_12 = - "2PACX-1vRCvn9X8LdOLYpiq_8U8Ihw8m_q0Lrl0Gkx4kJ22dhxX9Biy-Bhc0KWWxFQ9Fk2oS5pjgPNEd4I4XHD"; -const PUBLISHED_SHEET_ID_13 = - "2PACX-1vT6RKqvY0VzMaN7pKyYPyVXvUYR5cu3L5Z0sTeayDRE72xCXqVU-rhgyAucjGMJDDG5rXRKInPChqrJ"; -const PUBLISHED_SHEET_ID_14 = - "2PACX-1vQujsobkf1XNHg60LutKI8SXXITPGEtSx7F2sR0rBIm_FnFqXKfhz1MnZ1hIAyVAyhbPXbaf5NLXG-Q"; -const PUBLISHED_SHEET_ID_15 = - "2PACX-1vTsiPxkxMFJWmQQegSkpZgf3dNLqY7gc4msrnCbARgdNrr0wa9dbEDmtW9OzCrGeAmLDL1idbxU_gUk"; -const PUBLISHED_SHEET_ID_16 = - "2PACX-1vTDdjG51mUgQXFlBigDAF5QTpA9YL9XbhVZzjKSMqcsrD3dx9LeJfGdyBabsReECgyazhCNd3YOHQOa"; -const PUBLISHED_SHEET_ID_17 = - "2PACX-1vTWUT8wCTjJvROykckn6C30jNt2YVqS6zWyxKs4t0YtKfNAzJ7hxh7OggnZ3RjRokxYqSgvYEON9icz"; -const PUBLISHED_SHEET_ID_18 = - "2PACX-1vSDMdBWod4Db0eiA052qK1mtgzAAT7JnUeend__jvvPVKeo9Bhp8ur1Z4D41yiq8aGiAZkjsWn1KnZe"; -const PUBLISHED_SHEET_ID_19 = - "2PACX-1vQRxJ4zOl__ylTmlS5AnGKELJINLUElsRvJQp8YQBK7iPsjRzl1ApUwtgZJWeRYeQvlP2CCCeDuuFV2"; -const PUBLISHED_SHEET_ID_20 = - "2PACX-1vSeQYg9K6w1B9sfUAMfviYLh8r6n6uk9Bf4g7miDotpeV9j8bC_mZvco0Xr3J-zgv3aj7Xm8mc7GMAJ"; -const PUBLISHED_SHEET_ID_21 = - "2PACX-1vQT9ukpSK16ubwCFSSsZQHVlggAElw9999Rbtdsg9Opk_LyOCUDbMsAW5oiWxLjG4s0IudlLmhCVZuv"; -const PUBLISHED_SHEET_ID_22 = - "2PACX-1vTwNTdGXrAa-RHrvb-bE51qE2UMEVO9tb45o_W_djHESofOI5purP0hq4VlBl6Qr1u3F4uR8N7a0nmE"; -const PUBLISHED_SHEET_ID_23 = - "2PACX-1vRtezerCCxfVaYMerTEyPITsoDO_dx5UpTNcsU6YE8tTsmCflpHmx_0NZDKgPu47wB0_2vfIuxXopnF"; -const PUBLISHED_SHEET_ID_24 = - "2PACX-1vTVvDzqqG3GF1hyvQ9AAq7SXFr0FizVordgnL9kVAionTzwrgIM3pvMY6o1K8EoMnU1oiIJYZ2Ju6V_"; -const PUBLISHED_SHEET_ID_25 = - "2PACX-1vSJL0LHDI8dBKtTM3Q3mozy4AW3GUfunNSkkrdrftX3CgjQ3JyrxkqBU_6sr44T2wkZR6NMq2vp1UAf"; -const PUBLISHED_SHEET_ID_26 = - "2PACX-1vQGs1RLLszaHsZa59xaA-z-de1dZR0mBEX2K3GcJ_OSlE0zI05oTw22-nFcIUypcxpSKhEJaFG0I9SJ"; -const PUBLISHED_SHEET_ID_27 = - "2PACX-1vT5IUH2r83DCLWxBkfuiQOmYTi0Re41KYh2H7tq4_J-FPm82S7eDOeAmfY3rSfTyNysHMNL88EctL-N"; -const PUBLISHED_SHEET_ID_28 = - "2PACX-1vT2QMw9OyjcazyGXuHMj-jQ81oMnDDITU3q7fOLpcZ2-W81sNilWB0GdC5v2yv-0e0aMZnr-ZJAFowz"; -const PUBLISHED_SHEET_ID_29 = - "2PACX-1vTu2gcDkpfLS6g2_eYUtLBtQ3AaqC2UaSEHA20eCkgM41np7BZn0tE-mQ1ojcpr6t9R33WlTV4YHfve"; -const PUBLISHED_SHEET_ID_30 = - "2PACX-1vSH9k7EkLLVbnIk-whp4A9mx6WDoTQRVDVZGsHU812eHn65Uj08oL7i4kCaE2Mv01SLa1FuaHD35Nf4"; -const PUBLISHED_SHEET_ID_31 = - "2PACX-1vRboZ3VJ6RTk0md4CI9cwk-K2QwCkCTsnvMEa4rnK1uSd0WubZgo5hnfReaD2EZ0ZpQHzkRuckJrsHk"; -const PUBLISHED_SHEET_ID_32 = - "2PACX-1vTt1y5sQRZfLp0OjGSnDL59oNjDPsEKnrEIlSfpN491LM-f1dtoQGHfNwHFZSkW0-WShT6nWfGpdQzn"; -const PUBLISHED_SHEET_ID_33 = - "2PACX-1vRGEynR2mUrf7VNN0HgvKhAdtTRwLuHmnJ45YCzgjH7-iVZcoMnzgx-38O2nxT0dnH3sCQpdOXc2nJw"; -const PUBLISHED_SHEET_ID_34 = - "2PACX-1vSHmqvgLEi6R1roN1-SpW5B5Wei6myHLFmPplCJJor5x6g3GYquXH9td9niTqqGnya8L0u0axT9tWka"; -const PUBLISHED_SHEET_ID_35 = - "2PACX-1vRQqIlXr9zQfLH531tEsQNbqnNMv5a2MudN0BWrrWtVgWiT2VujA4JlJ5zbO8Oau6qDJ6tkD5KoycfU"; +const PUBLISHED_SHEET_ID_1 = "2PACX-1vSz8Qs1gE_IYpzlkFkCXGcL_BqR8hZieWVi-rphN1gfrO3H4lDtVZs4kd0C3P8Y9lhsT1rhoB-Q_cP4"; +const PUBLISHED_SHEET_ID_2 = "2PACX-1vRodtoTyQwXckfuvuQllkMhGC_gruigaaizVc8I6-BZWeetYpmRyexnO75ep7rnSxFICd8c9dfpwU8I"; +const PUBLISHED_SHEET_ID_3 = "2PACX-1vR_17UovavD4X7m_pqzmXjA_kCjGxIapemdWpRhDELHR1LbLJ-EVbxjKgeQat489BFRZ9bqMf-ILe_H"; +const PUBLISHED_SHEET_ID_4 = "2PACX-1vSeAoAk_iMv7cQ0tldZC7aivJmGKM5Wpc5VVr37Nzv-geTmtr6pDMb-oDK59RS21Om80-SYR3jRp6qq"; +const PUBLISHED_SHEET_ID_5 = "2PACX-1vSEikAgjAB9x7yhx4zNOUGLIx8Zfy2mAzRv0K1tbw08g73MO88-bbWCsgmhJ0uXa0gtuUlLMOnE9h26"; +const PUBLISHED_SHEET_ID_6 = "2PACX-1vQQmgjCktQknnTPy-s4OFycu-imtoMCrWY5M2Lqig3nhGyy6W5E27xbCyaaKV9lGaDWmTzGWVzPH9-S"; +const PUBLISHED_SHEET_ID_7 = "2PACX-1vR6blqV85tiBO-9u4MCW72qXALS3f7yQD0iV47MbsmIcKrvBDTorIVrUJ96QrxUj7iwAviYiecjp8VU"; +const PUBLISHED_SHEET_ID_8 = "2PACX-1vR1zl3JStozuCgPsPol19f9k_io1ABmHS_mOl9gzWxiDd2_WvWhdfhePXBFZIUFjpW-gPfPwE9m7AA_"; +const PUBLISHED_SHEET_ID_9 = "2PACX-1vRb4AsEPrV4b0S4j2vQku-J5XHnh8c_8fzmIhD2S2aMc2if7g6bLwJNYOPV8UmrrNR-Bv0C0yjcUnU3"; +const PUBLISHED_SHEET_ID_10 = "2PACX-1vQyBRow24Pc7Wm_mSjU3JDy_Ua5mFByz6zE7-vFguBvUOdcr-90PgNcTBOCL-nTa40WrghiAN-kSFVX"; +const PUBLISHED_SHEET_ID_11 = "2PACX-1vTd_tTI33CBI4obGaKTo0dfw1cNu5dUz4OIIhdbWJVmZJlEVslMyWzky1ifb9uRmV0siVxneBW4iBwi"; +const PUBLISHED_SHEET_ID_12 = "2PACX-1vRCvn9X8LdOLYpiq_8U8Ihw8m_q0Lrl0Gkx4kJ22dhxX9Biy-Bhc0KWWxFQ9Fk2oS5pjgPNEd4I4XHD"; +const PUBLISHED_SHEET_ID_13 = "2PACX-1vT6RKqvY0VzMaN7pKyYPyVXvUYR5cu3L5Z0sTeayDRE72xCXqVU-rhgyAucjGMJDDG5rXRKInPChqrJ"; +const PUBLISHED_SHEET_ID_14 = "2PACX-1vQujsobkf1XNHg60LutKI8SXXITPGEtSx7F2sR0rBIm_FnFqXKfhz1MnZ1hIAyVAyhbPXbaf5NLXG-Q"; +const PUBLISHED_SHEET_ID_15 = "2PACX-1vTsiPxkxMFJWmQQegSkpZgf3dNLqY7gc4msrnCbARgdNrr0wa9dbEDmtW9OzCrGeAmLDL1idbxU_gUk"; +const PUBLISHED_SHEET_ID_16 = "2PACX-1vTDdjG51mUgQXFlBigDAF5QTpA9YL9XbhVZzjKSMqcsrD3dx9LeJfGdyBabsReECgyazhCNd3YOHQOa"; +const PUBLISHED_SHEET_ID_17 = "2PACX-1vTWUT8wCTjJvROykckn6C30jNt2YVqS6zWyxKs4t0YtKfNAzJ7hxh7OggnZ3RjRokxYqSgvYEON9icz"; +const PUBLISHED_SHEET_ID_18 = "2PACX-1vSDMdBWod4Db0eiA052qK1mtgzAAT7JnUeend__jvvPVKeo9Bhp8ur1Z4D41yiq8aGiAZkjsWn1KnZe"; +const PUBLISHED_SHEET_ID_19 = "2PACX-1vQRxJ4zOl__ylTmlS5AnGKELJINLUElsRvJQp8YQBK7iPsjRzl1ApUwtgZJWeRYeQvlP2CCCeDuuFV2"; +const PUBLISHED_SHEET_ID_20 = "2PACX-1vSeQYg9K6w1B9sfUAMfviYLh8r6n6uk9Bf4g7miDotpeV9j8bC_mZvco0Xr3J-zgv3aj7Xm8mc7GMAJ"; +const PUBLISHED_SHEET_ID_21 = "2PACX-1vQT9ukpSK16ubwCFSSsZQHVlggAElw9999Rbtdsg9Opk_LyOCUDbMsAW5oiWxLjG4s0IudlLmhCVZuv"; +const PUBLISHED_SHEET_ID_22 = "2PACX-1vTwNTdGXrAa-RHrvb-bE51qE2UMEVO9tb45o_W_djHESofOI5purP0hq4VlBl6Qr1u3F4uR8N7a0nmE"; +const PUBLISHED_SHEET_ID_23 = "2PACX-1vRtezerCCxfVaYMerTEyPITsoDO_dx5UpTNcsU6YE8tTsmCflpHmx_0NZDKgPu47wB0_2vfIuxXopnF"; +const PUBLISHED_SHEET_ID_24 = "2PACX-1vTVvDzqqG3GF1hyvQ9AAq7SXFr0FizVordgnL9kVAionTzwrgIM3pvMY6o1K8EoMnU1oiIJYZ2Ju6V_"; +const PUBLISHED_SHEET_ID_25 = "2PACX-1vSJL0LHDI8dBKtTM3Q3mozy4AW3GUfunNSkkrdrftX3CgjQ3JyrxkqBU_6sr44T2wkZR6NMq2vp1UAf"; +const PUBLISHED_SHEET_ID_26 = "2PACX-1vQGs1RLLszaHsZa59xaA-z-de1dZR0mBEX2K3GcJ_OSlE0zI05oTw22-nFcIUypcxpSKhEJaFG0I9SJ"; +const PUBLISHED_SHEET_ID_27 = "2PACX-1vT5IUH2r83DCLWxBkfuiQOmYTi0Re41KYh2H7tq4_J-FPm82S7eDOeAmfY3rSfTyNysHMNL88EctL-N"; +const PUBLISHED_SHEET_ID_28 = "2PACX-1vT2QMw9OyjcazyGXuHMj-jQ81oMnDDITU3q7fOLpcZ2-W81sNilWB0GdC5v2yv-0e0aMZnr-ZJAFowz"; +const PUBLISHED_SHEET_ID_29 = "2PACX-1vTu2gcDkpfLS6g2_eYUtLBtQ3AaqC2UaSEHA20eCkgM41np7BZn0tE-mQ1ojcpr6t9R33WlTV4YHfve"; +const PUBLISHED_SHEET_ID_30 = "2PACX-1vSH9k7EkLLVbnIk-whp4A9mx6WDoTQRVDVZGsHU812eHn65Uj08oL7i4kCaE2Mv01SLa1FuaHD35Nf4"; +const PUBLISHED_SHEET_ID_31 = "2PACX-1vRboZ3VJ6RTk0md4CI9cwk-K2QwCkCTsnvMEa4rnK1uSd0WubZgo5hnfReaD2EZ0ZpQHzkRuckJrsHk"; +const PUBLISHED_SHEET_ID_32 = "2PACX-1vTt1y5sQRZfLp0OjGSnDL59oNjDPsEKnrEIlSfpN491LM-f1dtoQGHfNwHFZSkW0-WShT6nWfGpdQzn"; +const PUBLISHED_SHEET_ID_33 = "2PACX-1vRGEynR2mUrf7VNN0HgvKhAdtTRwLuHmnJ45YCzgjH7-iVZcoMnzgx-38O2nxT0dnH3sCQpdOXc2nJw"; +const PUBLISHED_SHEET_ID_34 = "2PACX-1vSHmqvgLEi6R1roN1-SpW5B5Wei6myHLFmPplCJJor5x6g3GYquXH9td9niTqqGnya8L0u0axT9tWka"; +const PUBLISHED_SHEET_ID_35 = "2PACX-1vT-ZBr4mrHY2PNiX_eG7BnOfBfLNbRNfvvgYlSZCfpVcJg-cGkB3X_yDaxDRDotgu7EOB_3j5e_msGv"; +const PUBLISHED_SHEET_ID_36 = "2PACX-1vTrt_V4yW0jd91chhz9BJZOgJtFrsaZEa_gPlrFfQToBuuNDDkn01w0K0GdnjCdklyzFz84A1hFbSUN"; +const PUBLISHED_SHEET_ID_37 = "2PACX-1vT1nasvyPHE2gVXu14urA6KHXuUzsRXtS3Qr6JORpa5Anm2TEr5KxJv11EvcE6UAx0Cj2Ja4j6IWR7g"; +const PUBLISHED_SHEET_ID_38 = "2PACX-1vRtf-9xJACCy0N6tqWQLEbTswAnX2h7_emrm2yh6-R9hw-AetGiH1PxRSJ9WzLfjAnF0Ci_JCSMvlPy"; - const SHEETS_V1 = [ ["raw_data1", "0"], ["death_and_recovered1", "200733542"], @@ -101,69 +68,41 @@ const SHEETS_V3 = [ ]; const SHEETS_V4 = [["raw_data4", "0"]]; - const SHEETS_V5 = [["raw_data5", "0"]]; - const SHEETS_V6 = [["raw_data6", "0"]]; - const SHEETS_V7 = [["raw_data7", "0"]]; - const SHEETS_V8 = [["raw_data8", "0"]]; - const SHEETS_V9 = [["raw_data9", "0"]]; - const SHEETS_V10 = [["raw_data10", "0"]]; - const SHEETS_V11 = [["raw_data11", "0"]]; - const SHEETS_V12 = [["raw_data12", "0"]]; - const SHEETS_V13 = [["raw_data13", "0"]]; - const SHEETS_V14 = [["raw_data14", "0"]]; - const SHEETS_V15 = [["raw_data15", "0"]]; - const SHEETS_V16 = [["raw_data16", "0"]]; - const SHEETS_V17 = [["raw_data17", "0"]]; - const SHEETS_V18 = [["raw_data18", "0"]]; - const SHEETS_V19 = [["raw_data19", "0"]]; - const SHEETS_V20 = [["raw_data20", "0"]]; - const SHEETS_V21 = [["raw_data21", "0"]]; - const SHEETS_V22 = [["raw_data22", "0"]]; - const SHEETS_V23 = [["raw_data23", "0"]]; - const SHEETS_V24 = [["raw_data24", "0"]]; - const SHEETS_V25 = [["raw_data25", "0"]]; - const SHEETS_V26 = [["raw_data26", "0"]]; - const SHEETS_V27 = [["raw_data27", "0"]]; - const SHEETS_V28 = [["raw_data28", "0"]]; - const SHEETS_V29 = [["raw_data29", "0"]]; - const SHEETS_V30 = [["raw_data30", "0"]]; - const SHEETS_V31 = [["raw_data31", "0"]]; - const SHEETS_V32 = [["raw_data32", "0"]]; - const SHEETS_V33 = [["raw_data33", "0"]]; - const SHEETS_V34 = [["raw_data34", "0"]]; - -const SHEETS_V35 = [ - ["raw_data35", "0"], +const SHEETS_V35 = [["raw_data35", "0"]]; +const SHEETS_V36 = [["raw_data36", "0"]]; +const SHEETS_V37 = [["raw_data37", "0"]]; +const SHEETS_V38 = [ + ["raw_data38", "0"], ["state_wise", "1896310216"], ["state_wise_daily", "1395461826"], ["sources_list", "704389477"], @@ -246,6 +185,9 @@ async function sheetsToCSV(sheets, pubId) { // await sheetsToCSV(SHEETS_V31, PUBLISHED_SHEET_ID_31); // await sheetsToCSV(SHEETS_V32, PUBLISHED_SHEET_ID_32); // await sheetsToCSV(SHEETS_V33, PUBLISHED_SHEET_ID_33); - await sheetsToCSV(SHEETS_V34, PUBLISHED_SHEET_ID_34); - await sheetsToCSV(SHEETS_V35, PUBLISHED_SHEET_ID_35); + // await sheetsToCSV(SHEETS_V34, PUBLISHED_SHEET_ID_34); + // await sheetsToCSV(SHEETS_V35, PUBLISHED_SHEET_ID_35); + // await sheetsToCSV(SHEETS_V36, PUBLISHED_SHEET_ID_36); + await sheetsToCSV(SHEETS_V37, PUBLISHED_SHEET_ID_37); + await sheetsToCSV(SHEETS_V38, PUBLISHED_SHEET_ID_38); })();