Update solver for PUF stage 2 from PuLP to CVXOPT

.gitignore

@@ -21,3 +21,8 @@ cps_data/pycps/cps_raw.csv.gz
 cpsmar*.sas
 cpsmar*.csv
 *.dat
+
+# pickle files
+cps_data/pycps/data/cpsmar2013.pkl
+cps_data/pycps/data/cpsmar2014.pkl
+cps_data/pycps/data/cpsmar2015.pkl

Makefile

@@ -72,10 +72,10 @@ git-pr:
 	@./gitpr $(N)
 
 .PHONY=puf-files
-puf-files: puf_data/puf.csv \
-           puf_stage1/growfactors.csv \
-           puf_stage2/puf_weights.csv.gz \
-           puf_stage3/puf_ratios.csv
+puf-files:  puf_data/puf.csv \
+            puf_stage1/growfactors.csv \
+            puf_stage2/puf_weights.csv.gz \
+            puf_stage3/puf_ratios.csv
 
 PM_DIR=./puf_data/StatMatch/Matching
 PM_PY_FILES := $(shell ls -l $(PM_DIR)/*py | awk '{print $$9}')
@@ -88,7 +88,7 @@ puf_data/puf.csv: puf_data/finalprep.py \
                   puf_data/impute_itmexp.py \
                   puf_data/impute_pencon.py \
                   puf_data/cps-matched-puf.csv
-	cd puf_data ; python finalprep.py
+	cd puf_data ;  python finalprep.py
 
 puf_stage1/Stage_I_factors.csv: puf_stage1/stage1.py \
                                 puf_stage1/CBO_baseline.csv \
@@ -97,54 +97,64 @@ puf_stage1/Stage_I_factors.csv: puf_stage1/stage1.py \
                                 puf_stage1/NP2014_D1.csv \
                                 puf_stage1/SOI_estimates.csv \
                                 puf_stage1/US-EST00INT-ALLDATA.csv
-	cd puf_stage1 ; python stage1.py
+	cd puf_stage1 ;  python stage1.py
 # above recipe also makes puf_stage1/Stage_II_targets.csv
 
 puf_stage1/growfactors.csv: puf_stage1/factors_finalprep.py \
                             puf_stage1/Stage_I_factors.csv \
                             puf_stage1/benefit_growth_rates.csv
-	cd puf_stage1 ; python factors_finalprep.py
+	cd puf_stage1 ;  python factors_finalprep.py
 
-puf_stage2/puf_weights.csv.gz: puf_stage2/stage2.py \
-                               puf_stage2/solve_lp_for_year.py \
-                               puf_data/cps-matched-puf.csv \
-                               puf_stage1/Stage_I_factors.csv \
-                               puf_stage1/Stage_II_targets.csv
-	cd puf_stage2 ; python stage2.py && \
+puf_stage2/puf_weights.csv.gz:  puf_stage2/stage2.py \
+                                puf_stage2/solve_lp_for_year.py \
+                                puf_data/cps-matched-puf.csv \
+                                puf_stage1/Stage_I_factors.csv \
+                                puf_stage1/Stage_II_targets.csv
+	cd puf_stage2 ;  python stage2.py && \
         gunzip puf_weights.csv.gz && gzip -n puf_weights.csv
 
-puf_stage3/puf_ratios.csv: puf_stage3/stage3.py \
-                           puf_stage3/stage3_targets.csv \
-                           puf_data/cps-matched-puf.csv \
-                           puf_stage1/growfactors.csv \
-                           puf_stage2/puf_weights.csv.gz
-	cd puf_stage3 ; python stage3.py
+puf_stage3/puf_ratios.csv:  puf_stage3/stage3.py \
+                            puf_stage3/stage3_targets.csv \
+                            puf_data/cps-matched-puf.csv \
+                            puf_stage1/growfactors.csv \
+                            puf_stage2/puf_weights.csv.gz
+	cd puf_stage3 ;  python stage3.py
 
 .PHONY=cps-files
-cps-files: cps_data/cps.csv.gz \
+cps-files: cps_data/pycps/cps_raw.csv.gz \
            cps_stage1/stage_2_targets.csv \
            cps_stage2/cps_weights.csv.gz
 
-cps_data/cps.csv.gz: cps_data/finalprep.py \
-                     cps_data/cps_raw.csv.gz \
-                     cps_data/adjustment_targets.csv \
-                     cps_data/benefitprograms.csv
-	cd cps_data ; python finalprep.py && \
-        gunzip cps.csv.gz && gzip -n cps.csv
+cps_data/pycps/cps_raw.csv.gz:  cps_data/pycps/create.py \
+                                cps_data/pycps/benefits.py \
+                                cps_data/pycps/filing_rules.json \
+                                cps_data/pycps/finalprep.py \
+                                cps_data/pycps/helpers.py \
+                                cps_data/pycps/impute.py \
+                                cps_data/pycps/pycps.py \
+                                cps_data/pycps/splitincome.py \
+                                cps_data/pycps/targeting.py \
+                                cps_data/pycps/taxunit.py \
+                                cps_data/pycps/template.txt \
+                                cps_data/pycps/transform_sas.py \
+                                cps_data/pycps/adjustment_targets.csv \
+                                cps_data/benefitprograms.csv
+	cd cps_data/pycps ;  python create.py && \
+	gunzip cps.csv.gz && gzip -n cps.csv
 
 cps_stage1/stage_2_targets.csv: cps_stage1/stage1.py \
                                 cps_stage1/SOI_estimates.csv \
                                 puf_stage1/Stage_I_factors.csv \
                                 puf_stage1/Stage_II_targets.csv
-	cd cps_stage1 ; python stage1.py
+	cd cps_stage1 ;  python stage1.py
 
 cps_stage2/cps_weights.csv.gz: cps_stage2/stage2.py \
                                cps_stage2/solve_lp_for_year.py \
-                               cps_data/cps_raw.csv.gz \
+                               cps_data/pycps/cps_raw.csv.gz \
                                puf_stage1/Stage_I_factors.csv \
                                cps_stage1/stage_2_targets.csv
-	cd cps_stage2 ; python stage2.py && \
-        gunzip cps_weights.csv.gz && gzip -n cps_weights.csv
+	cd cps_stage2 ;  python stage2.py && \
+	gunzip cps_weights.csv.gz && gzip -n cps_weights.csv
 
 .PHONY=all
-all: puf-files cps-files
+all:  puf-files cps-files

README.md

@@ -35,6 +35,21 @@ sample data being used.  But the weights, ratios, and benefits files
 do depend on the data file, so they are different in the two sets of
 data files.
 
+Installation
+-----------
+
+Currently, the only way to install `taxdata` currently is to clone the git
+repo locally.
+```
+git clone https://github.com/PSLmodels/taxdata.git
+```
+Next navigate to the directory and install the `taxdata-dev` conda environment
+```
+cd taxdata
+conda env create -f environment.yml
+```
+To run the scripts that produce `puf.csv` and `cps.csv.gz`, activate the
+`taxdata-dev` conda environment and follow the workflow laid out below.
 
 Data-Preparation Documentation and Workflow
 -------------------------------------------
@@ -51,18 +66,8 @@ running the `make help` command.  If you want more background on the
 make utility and makefiles, search for Internet links with the
 keywords `makefile` and `automate`.
 
-Note that the stage2 linear program that generates the weights file is
-very long-running, taking five or more hours depending on your
-computer's CPU speed.  We are considering options for speeding up this
-stage2 work, but for the time being you can execute `make puf-files`
-and `make cps-files` in separate terminal windows to have the two
-stage2 linear programs run in parallel.  (If you try this parallel
-execution approach, be sure to wait for the `make puf-files` job to
-begin stage2 work before executing the `make cps-files` command in
-the other terminal window.  This is necessary because the CPS stage1
-work depends on output from PUF stage1.)  If you are generating the
-taxdata made files in an overnight run, then simply execute the `make
-all` command.
+The stage 2 linear program that generate weights for the PUF and CPS files take about 20 minutes each to run, but if you'd like to slightly speed up the process, you can you can execute `make puf-files` and `make cps-files` in separate terminal windows to have both stage 2 linear programs run in parallel. In order for this parallel execution to run smoothly, please wait for `make puf-files` to begin stage 2 work before executing the `make cps-files` command in
+the other terminal window. This is necessary because the CPS stage1 work depends on output from PUF stage1. If you are generating the taxdata made files in an overnight run, then simply execute the `make all` command.
 
 You can copy the made files to your local Tax-Calculator directory
 tree using the [`csvcopy.sh`](csvcopy.sh) bash script.  Use the `dryrun`
@@ -71,6 +76,14 @@ the corresponding files in the Tax-Calculator directory tree) without
 actually doing the file copies.  At the terminal command-prompt in the
 top-level taxdata directory, execute `./csvcopy.sh` to get help.
 
+### Example
+
+To create `cps.csv.gz`, run
+```
+conda activate taxdata-dev
+make cps-files
+```
+
 
 Contributing to taxdata Repository
 ----------------------------------
@@ -81,9 +94,17 @@ to make sure your proposed code is consistent with the repository's
 coding style and then run `make pytest` to ensure that all the tests
 pass.
 
+Disclaimer
+----------
+
+`taxdata` is under continuous development. As such, results will change as the
+underlying data and logic improves.
+
 
 Contributors
 ------------
-- Anderson Frailey
-- John O'Hare
-- Amy Xu
+
+A full list of contributors on GitHub can be found 
+here](https://github.com/PSLmodels/taxdata/graphs/contributors). John O'Hare
+of Quantria Strategies has also made significant contributions to the
+development of `taxdata`.