Fit with theory covmat with n3fit

doc/sphinx/source/vp/theorycov/index.rst

@@ -29,7 +29,7 @@ Summary
 -  Theoretical covariance matrices are built according to the various prescriptions
    in :ref:`prescrips`. 
 
--  The prescription must be one of 3 point, 5 point, 5bar point, 7 point or 9 point. You can specify
+-  The prescription must be one of 3 point, 3r point, 3f point, 5 point, 5bar point, 7 point or 9 point. You can specify
    this using ``point_prescription: "x point"`` in the runcard. The translation of this flag 
    into the relevant ``theoryids`` is handled by the ``scalevariations`` module in ``validphys``.
 

doc/sphinx/source/vp/theorycov/runcard_layout.rst

@@ -1,11 +1,6 @@
 Important information about runcard layout
 ==========================================
 
--  The flag ``fivetheories`` specifies the choice of 5 or
-   :math:`\bar{5}` prescription for the case of 5 input theories. You
-   must assign a value ``nobar`` or ``bar`` correspondingly. If you do
-   not do this, ``validphys`` will give an error.
-
 -  The default behaviour for the 7-point prescription is to use Gavin
    Salam's modification to it. To use the original 7-point prescription
    instead, the ``seventheories`` flag must be set to ``original``.

n3fit/src/n3fit/performfit.py

@@ -37,7 +37,7 @@ def performfit(
     tensorboard=None,
     debug=False,
     maxcores=None,
-    parallel_models=False
+    parallel_models=False,
 ):
     """
         This action will (upon having read a validcard) process a full PDF fit
@@ -148,7 +148,6 @@ def performfit(
     #       [list of all NN seeds]
     #       )
     #
-
     n_models = len(replicas_nnseed_fitting_data_dict)
     if parallel_models and n_models != 1:
         replicas, replica_experiments, nnseeds = zip(*replicas_nnseed_fitting_data_dict)

n3fit/src/n3fit/scripts/n3fit_exec.py

@@ -145,7 +145,13 @@ def from_yaml(cls, o, *args, **kwargs):
             validation_action = namespace + "validation_pseudodata"
 
             N3FIT_FIXED_CONFIG['actions_'].extend((training_action, validation_action))
-
+        N3FIT_FIXED_CONFIG['theory_covmat_flag'] = False
+        N3FIT_FIXED_CONFIG['use_user_uncertainties'] = None
+        N3FIT_FIXED_CONFIG['use_scalevar_uncertainties'] = None
+        if file_content.get('theorycovmatconfig') is not None:
+            N3FIT_FIXED_CONFIG['theory_covmat_flag'] = True
+            N3FIT_FIXED_CONFIG['use_user_uncertainties'] = file_content.get('theorycovmatconfig').get('use_user_uncertainties')
+            N3FIT_FIXED_CONFIG['use_scalevar_uncertainties'] = file_content.get('theorycovmatconfig').get('use_scalevar_uncertainties')
         file_content.update(N3FIT_FIXED_CONFIG)
         return cls(file_content, *args, **kwargs)
 

n3fit/src/n3fit/scripts/vp_setupfit.py

@@ -158,7 +158,6 @@ def from_yaml(cls, o, *args, **kwargs):
             SETUPFIT_FIXED_CONFIG['actions_'] += [filter_action]
         else:
             SETUPFIT_FIXED_CONFIG['actions_'] += [check_n3fit_action, filter_action]
-
         if file_content.get('theorycovmatconfig') is not None:
             SETUPFIT_FIXED_CONFIG['actions_'].append(
                 'datacuts::theory::theorycovmatconfig nnfit_theory_covmat')

validphys2/src/validphys/config.py

@@ -1053,11 +1053,9 @@ def produce_nnfit_theory_covmat(
             # Only user uncertainties
             from validphys.theorycovariance.construction import user_covmat_fitting
             f = user_covmat_fitting
-
         @functools.wraps(f)
         def res(*args, **kwargs):
             return f(*args, **kwargs)
-
         # Set this to get the same filename regardless of the action.
         res.__name__ = "theory_covmat"
         return res
@@ -1504,6 +1502,11 @@ def produce_group_dataset_inputs_by_metadata(
             {"data_input": NSList(group, nskey="dataset_input"), "group_name": name}
             for name, group in res.items()
         ]
+
+    def produce_group_dataset_inputs_by_fitting_group(self, data_input, theory_covmat_flag):
+        if theory_covmat_flag is True:
+            return self.produce_group_dataset_inputs_by_metadata(data_input, "custom_group")
+        return self.produce_group_dataset_inputs_by_metadata(data_input, "experiment")
 
 
     def produce_group_dataset_inputs_by_experiment(self, data_input):
@@ -1512,6 +1515,7 @@ def produce_group_dataset_inputs_by_experiment(self, data_input):
     def produce_group_dataset_inputs_by_process(self, data_input):
         return self.produce_group_dataset_inputs_by_metadata(data_input, "nnpdf31_process")
 
+
     def produce_scale_variation_theories(self, theoryid, point_prescription):
         """Produces a list of theoryids given a theoryid at central scales and a point
            prescription. The options for the latter are '3 point', '5 point', '5bar point', '7 point'

validphys2/src/validphys/covmats.py

@@ -6,6 +6,7 @@
 import numpy as np
 import pandas as pd
 import scipy.linalg as la
+import pathlib
 
 from reportengine import collect
 from reportengine.table import table
@@ -23,7 +24,7 @@
 from validphys.core import PDF, DataGroupSpec, DataSetSpec
 from validphys.covmats_utils import construct_covmat, systematics_matrix
 from validphys.results import ThPredictionsResult
-
+from validphys.commondata import loaded_commondata_with_cuts
 log = logging.getLogger(__name__)
 
 INTRA_DATASET_SYS_NAME = ("UNCORR", "CORR", "THEORYUNCORR", "THEORYCORR")
@@ -227,6 +228,17 @@ def dataset_inputs_covmat_from_systematics(
             covmat,
             norm_threshold=norm_threshold
         )
+   # try:
+   #     theory_covmat_path = pathlib.Path.cwd()
+   #     data = pd.read_csv(theory_covmat_path / "prov_moredata" / "tables" / "datacuts_theory_theorycovmatconfig_theory_covmat_custom.csv", sep='\t')
+   #     datael = data.iloc[3:]
+    #    datael = datael.drop(['group'], axis=1)
+    #    datael = datael.drop(['Unnamed: 1'], axis=1)
+    #    datael = datael.drop(['Unnamed: 2'], axis=1)
+    #    theory_covmat = np.copy(datael.values)
+    #except FileNotFoundError:
+    #    theory_covmat = np.zeros(covmat.shape) 
+    #total_covmat = np.add(covmat, theory_covmat)   
     return covmat
 
 
@@ -339,6 +351,41 @@ def dataset_inputs_t0_covmat_from_systematics(
         _list_of_central_values=dataset_inputs_t0_predictions
     )
 
+def dataset_inputs_t0_total_covmat(dataset_inputs_loaded_cd_with_cuts,
+    *,
+    data_input,
+    use_weights_in_covmat=True,
+    norm_threshold=None,
+    dataset_inputs_t0_predictions,
+    output_path,
+    theory_covmat_flag,
+    use_user_uncertainties,
+    use_scalevar_uncertainties):
+    exp_covmat = dataset_inputs_covmat_from_systematics(
+        dataset_inputs_loaded_cd_with_cuts,
+        data_input,
+        use_weights_in_covmat,
+        norm_threshold=norm_threshold,
+        _list_of_central_values=dataset_inputs_t0_predictions
+    )
+    if theory_covmat_flag is True:
+        generic_path = None
+        if use_scalevar_uncertainties is True:
+            if use_user_uncertainties is True:
+                generic_path = pathlib.Path("datacuts_theory_theorycovmatconfig_total_theory_covmat.csv")
+            else:
+                generic_path = pathlib.Path("datacuts_theory_theorycovmatconfig_theory_covmat_custom.csv")
+        else:
+            if use_user_uncertainties is True:
+                generic_path = pathlib.Path("datacuts_theory_theorycovmatconfig_user_covmat.csv")
+            else:
+                generic_path = pathlib.Path("datacuts_theory_theorycovmatconfig_theory_covmat_custom.csv")
+        theorypath = pathlib.Path(str(output_path/"tables"/generic_path.relative_to(generic_path.anchor)))
+        theory_covmat = pd.read_csv(theorypath, sep='\t')
+        theory_covmat = theory_covmat.iloc[3:].drop(['group'], axis=1).drop(['Unnamed: 1'], axis=1).drop(['Unnamed: 2'], axis=1)
+        return np.add(exp_covmat,theory_covmat.values.astype(np.float))
+    return exp_covmat
+
 
 def sqrt_covmat(covariance_matrix):
     """Function that computes the square root of the covariance matrix.

validphys2/src/validphys/n3fit_data.py

@@ -188,7 +188,7 @@ def _mask_fk_tables(dataset_dicts, tr_masks):
 def fitting_data_dict(
     data,
     make_replica,
-    dataset_inputs_t0_covmat_from_systematics,
+    dataset_inputs_t0_total_covmat,
     tr_masks,
     kfold_masks,
     diagonal_basis=None,
@@ -244,7 +244,7 @@ def fitting_data_dict(
     datasets = common_data_reader_experiment(spec_c, data)
 
     # t0 covmat
-    covmat = dataset_inputs_t0_covmat_from_systematics
+    covmat = dataset_inputs_t0_total_covmat
     inv_true = np.linalg.inv(covmat)
 
     if diagonal_basis:
@@ -297,7 +297,6 @@ def fitting_data_dict(
         folds["training"].append(fold[tr_mask])
         folds["validation"].append(fold[vl_mask])
         folds["experimental"].append(~fold)
-
     dict_out = {
         "datasets": datasets_copy,
         "name": str(data),
@@ -320,7 +319,7 @@ def fitting_data_dict(
     }
     return dict_out
 
-exps_fitting_data_dict = collect("fitting_data_dict", ("group_dataset_inputs_by_experiment",))
+exps_fitting_data_dict = collect("fitting_data_dict", ("group_dataset_inputs_by_fitting_group",))
 
 def replica_nnseed_fitting_data_dict(replica, exps_fitting_data_dict, replica_nnseed):
     """For a single replica return a tuple of the inputs to this function.