diff --git a/flowsa/__init__.py b/flowsa/__init__.py index f004184a4..530747b52 100644 --- a/flowsa/__init__.py +++ b/flowsa/__init__.py @@ -54,25 +54,30 @@ def getFlowByActivity(datasource, year, flowclass=None, geographic_level=None, fba = load_preprocessed_output(fba_meta, paths) # If that didn't work, try to download a remote version of FBA if fba is None and download_FBA_if_missing: - log.info('%s %s not found in %s, downloading from remote source', - datasource, str(year), fbaoutputpath) + log.info(f'{datasource} {str(year)} not found in {fbaoutputpath}, ' + 'downloading from remote source') download_from_remote(fba_meta, paths) fba = load_preprocessed_output(fba_meta, paths) # If that didn't work or wasn't allowed, try to construct the FBA if fba is None: - log.info('%s %s not found in %s, running functions to generate FBA', - datasource, str(year), fbaoutputpath) + log.info(f'{datasource} {str(year)} not found in {fbaoutputpath}, ' + 'running functions to generate FBA') # Generate the fba flowsa.flowbyactivity.main(year=year, source=datasource) # Now load the fba fba = load_preprocessed_output(fba_meta, paths) # If none of the above worked, log an error message if fba is None: - log.error('getFlowByActivity failed, FBA not found') + raise flowsa.exceptions.FBANotAvailableError(method=datasource, + year=year) # Otherwise (that is, if one of the above methods successfuly loaded the # FBA), log it. else: - log.info('Loaded %s %s from %s', datasource, str(year), fbaoutputpath) + log.info(f'Loaded {datasource} {str(year)} from {fbaoutputpath}') + + if len(fba) ==0: + raise flowsa.exceptions.FBANotAvailableError( + message=f"Error generating {datasource} for {str(year)}") # Address optional parameters if flowclass is not None: diff --git a/flowsa/common.py b/flowsa/common.py index d9c5570a6..6bdff2721 100644 --- a/flowsa/common.py +++ b/flowsa/common.py @@ -12,6 +12,7 @@ from dotenv import load_dotenv from esupy.processed_data_mgmt import create_paths_if_missing import flowsa.flowsa_yaml as flowsa_yaml +import flowsa.exceptions from flowsa.schema import flow_by_activity_fields, flow_by_sector_fields, \ flow_by_sector_collapsed_fields, flow_by_activity_mapped_fields, \ flow_by_activity_wsec_fields, flow_by_activity_mapped_wsec_fields, \ @@ -51,8 +52,7 @@ def load_api_key(api_source): load_dotenv(f'{MODULEPATH}API_Keys.env', verbose=True) key = os.getenv(api_source) if key is None: - log.error(f"Key file {api_source} not found. See github wiki for help " - "https://github.com/USEPA/flowsa/wiki/Using-FLOWSA#api-keys") + raise flowsa.exceptions.APIError(api_source=api_source) return key @@ -131,9 +131,9 @@ def load_yaml_dict(filename, flowbytype=None, filepath=None): try: with open(yaml_path, 'r') as f: config = flowsa_yaml.load(f, filepath) - except IOError: - log.error(f'{flowbytype} method file not found') - raise + except FileNotFoundError: + raise flowsa.exceptions.FlowsaMethodNotFoundError( + method_type=flowbytype, method=filename) return config diff --git a/flowsa/data/activitytosectormapping/NAICS_Crosswalk_BEA_2012_Summary.csv b/flowsa/data/activitytosectormapping/NAICS_Crosswalk_BEA_2012_Summary.csv new file mode 100644 index 000000000..9179537c3 --- /dev/null +++ b/flowsa/data/activitytosectormapping/NAICS_Crosswalk_BEA_2012_Summary.csv @@ -0,0 +1,1117 @@ +ActivitySourceName,Activity,SectorSourceName,Sector,SectorType +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111110, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111120, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111130, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111140, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111150, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111160, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111191, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111199, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111211, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111219, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111310, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111320, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111331, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111332, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111333, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111334, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111335, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111336, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111339, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111411, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111419, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111421, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111422, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111910, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111920, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111930, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111940, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111991, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111992, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,111998, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,112111, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,112112, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,112120, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,112130, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,112210, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,112310, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,112320, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,112330, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,112340, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,112390, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,112410, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,112420, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,112511, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,112512, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,112519, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,112910, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,112920, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,112930, +BEA_2012_Summary_Code,111CA,NAICS_2012_Code,112990, +BEA_2012_Summary_Code,113FF,NAICS_2012_Code,113110, +BEA_2012_Summary_Code,113FF,NAICS_2012_Code,113210, +BEA_2012_Summary_Code,113FF,NAICS_2012_Code,113310, +BEA_2012_Summary_Code,113FF,NAICS_2012_Code,114111, +BEA_2012_Summary_Code,113FF,NAICS_2012_Code,114112, +BEA_2012_Summary_Code,113FF,NAICS_2012_Code,114119, +BEA_2012_Summary_Code,113FF,NAICS_2012_Code,114210, +BEA_2012_Summary_Code,113FF,NAICS_2012_Code,115111, +BEA_2012_Summary_Code,113FF,NAICS_2012_Code,115112, +BEA_2012_Summary_Code,113FF,NAICS_2012_Code,115113, +BEA_2012_Summary_Code,113FF,NAICS_2012_Code,115114, +BEA_2012_Summary_Code,113FF,NAICS_2012_Code,115115, +BEA_2012_Summary_Code,113FF,NAICS_2012_Code,115116, +BEA_2012_Summary_Code,113FF,NAICS_2012_Code,115210, +BEA_2012_Summary_Code,113FF,NAICS_2012_Code,115310, +BEA_2012_Summary_Code,211,NAICS_2012_Code,211111, +BEA_2012_Summary_Code,211,NAICS_2012_Code,211112, +BEA_2012_Summary_Code,212,NAICS_2012_Code,212111, +BEA_2012_Summary_Code,212,NAICS_2012_Code,212112, +BEA_2012_Summary_Code,212,NAICS_2012_Code,212113, +BEA_2012_Summary_Code,212,NAICS_2012_Code,212210, +BEA_2012_Summary_Code,212,NAICS_2012_Code,212221, +BEA_2012_Summary_Code,212,NAICS_2012_Code,212222, +BEA_2012_Summary_Code,212,NAICS_2012_Code,212231, +BEA_2012_Summary_Code,212,NAICS_2012_Code,212234, +BEA_2012_Summary_Code,212,NAICS_2012_Code,212291, +BEA_2012_Summary_Code,212,NAICS_2012_Code,212299, +BEA_2012_Summary_Code,212,NAICS_2012_Code,212311, +BEA_2012_Summary_Code,212,NAICS_2012_Code,212312, +BEA_2012_Summary_Code,212,NAICS_2012_Code,212313, +BEA_2012_Summary_Code,212,NAICS_2012_Code,212319, +BEA_2012_Summary_Code,212,NAICS_2012_Code,212321, +BEA_2012_Summary_Code,212,NAICS_2012_Code,212322, +BEA_2012_Summary_Code,212,NAICS_2012_Code,212324, +BEA_2012_Summary_Code,212,NAICS_2012_Code,212325, +BEA_2012_Summary_Code,212,NAICS_2012_Code,212391, +BEA_2012_Summary_Code,212,NAICS_2012_Code,212392, +BEA_2012_Summary_Code,212,NAICS_2012_Code,212393, +BEA_2012_Summary_Code,212,NAICS_2012_Code,212399, +BEA_2012_Summary_Code,213,NAICS_2012_Code,213111, +BEA_2012_Summary_Code,213,NAICS_2012_Code,213112, +BEA_2012_Summary_Code,213,NAICS_2012_Code,213113, +BEA_2012_Summary_Code,213,NAICS_2012_Code,213114, +BEA_2012_Summary_Code,213,NAICS_2012_Code,213115, +BEA_2012_Summary_Code,22,NAICS_2012_Code,221111, +BEA_2012_Summary_Code,22,NAICS_2012_Code,221112, +BEA_2012_Summary_Code,22,NAICS_2012_Code,221113, +BEA_2012_Summary_Code,22,NAICS_2012_Code,221114, +BEA_2012_Summary_Code,22,NAICS_2012_Code,221115, +BEA_2012_Summary_Code,22,NAICS_2012_Code,221116, +BEA_2012_Summary_Code,22,NAICS_2012_Code,221117, +BEA_2012_Summary_Code,22,NAICS_2012_Code,221118, +BEA_2012_Summary_Code,22,NAICS_2012_Code,221121, +BEA_2012_Summary_Code,22,NAICS_2012_Code,221122, +BEA_2012_Summary_Code,22,NAICS_2012_Code,221210, +BEA_2012_Summary_Code,22,NAICS_2012_Code,221310, +BEA_2012_Summary_Code,22,NAICS_2012_Code,221320, +BEA_2012_Summary_Code,22,NAICS_2012_Code,221330, +BEA_2012_Summary_Code,23,NAICS_2012_Code,236115, +BEA_2012_Summary_Code,23,NAICS_2012_Code,236116, +BEA_2012_Summary_Code,23,NAICS_2012_Code,236117, +BEA_2012_Summary_Code,23,NAICS_2012_Code,236118, +BEA_2012_Summary_Code,23,NAICS_2012_Code,236210, +BEA_2012_Summary_Code,23,NAICS_2012_Code,236220, +BEA_2012_Summary_Code,23,NAICS_2012_Code,237110, +BEA_2012_Summary_Code,23,NAICS_2012_Code,237120, +BEA_2012_Summary_Code,23,NAICS_2012_Code,237130, +BEA_2012_Summary_Code,23,NAICS_2012_Code,237210, +BEA_2012_Summary_Code,23,NAICS_2012_Code,237310, +BEA_2012_Summary_Code,23,NAICS_2012_Code,237990, +BEA_2012_Summary_Code,23,NAICS_2012_Code,238110, +BEA_2012_Summary_Code,23,NAICS_2012_Code,238120, +BEA_2012_Summary_Code,23,NAICS_2012_Code,238130, +BEA_2012_Summary_Code,23,NAICS_2012_Code,238140, +BEA_2012_Summary_Code,23,NAICS_2012_Code,238150, +BEA_2012_Summary_Code,23,NAICS_2012_Code,238160, +BEA_2012_Summary_Code,23,NAICS_2012_Code,238170, +BEA_2012_Summary_Code,23,NAICS_2012_Code,238190, +BEA_2012_Summary_Code,23,NAICS_2012_Code,238210, +BEA_2012_Summary_Code,23,NAICS_2012_Code,238220, +BEA_2012_Summary_Code,23,NAICS_2012_Code,238290, +BEA_2012_Summary_Code,23,NAICS_2012_Code,238310, +BEA_2012_Summary_Code,23,NAICS_2012_Code,238320, +BEA_2012_Summary_Code,23,NAICS_2012_Code,238330, +BEA_2012_Summary_Code,23,NAICS_2012_Code,238340, +BEA_2012_Summary_Code,23,NAICS_2012_Code,238350, +BEA_2012_Summary_Code,23,NAICS_2012_Code,238390, +BEA_2012_Summary_Code,23,NAICS_2012_Code,238910, +BEA_2012_Summary_Code,23,NAICS_2012_Code,238990, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311111, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311119, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311211, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311212, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311213, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311221, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311224, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311225, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311230, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311313, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311314, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311340, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311351, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311352, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311411, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311412, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311421, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311422, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311423, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311511, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311512, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311513, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311514, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311520, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311611, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311612, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311613, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311615, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311710, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311811, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311812, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311813, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311821, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311824, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311830, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311911, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311919, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311920, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311930, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311941, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311942, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311991, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,311999, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,312111, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,312112, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,312113, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,312120, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,312130, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,312140, +BEA_2012_Summary_Code,311FT,NAICS_2012_Code,312230, +BEA_2012_Summary_Code,313TT,NAICS_2012_Code,313110, +BEA_2012_Summary_Code,313TT,NAICS_2012_Code,313210, +BEA_2012_Summary_Code,313TT,NAICS_2012_Code,313220, +BEA_2012_Summary_Code,313TT,NAICS_2012_Code,313230, +BEA_2012_Summary_Code,313TT,NAICS_2012_Code,313240, +BEA_2012_Summary_Code,313TT,NAICS_2012_Code,313310, +BEA_2012_Summary_Code,313TT,NAICS_2012_Code,313320, +BEA_2012_Summary_Code,313TT,NAICS_2012_Code,314110, +BEA_2012_Summary_Code,313TT,NAICS_2012_Code,314120, +BEA_2012_Summary_Code,313TT,NAICS_2012_Code,314910, +BEA_2012_Summary_Code,313TT,NAICS_2012_Code,314994, +BEA_2012_Summary_Code,313TT,NAICS_2012_Code,314999, +BEA_2012_Summary_Code,315AL,NAICS_2012_Code,315110, +BEA_2012_Summary_Code,315AL,NAICS_2012_Code,315190, +BEA_2012_Summary_Code,315AL,NAICS_2012_Code,315210, +BEA_2012_Summary_Code,315AL,NAICS_2012_Code,315220, +BEA_2012_Summary_Code,315AL,NAICS_2012_Code,315240, +BEA_2012_Summary_Code,315AL,NAICS_2012_Code,315280, +BEA_2012_Summary_Code,315AL,NAICS_2012_Code,315990, +BEA_2012_Summary_Code,315AL,NAICS_2012_Code,316110, +BEA_2012_Summary_Code,315AL,NAICS_2012_Code,316210, +BEA_2012_Summary_Code,315AL,NAICS_2012_Code,316992, +BEA_2012_Summary_Code,315AL,NAICS_2012_Code,316998, +BEA_2012_Summary_Code,321,NAICS_2012_Code,321113, +BEA_2012_Summary_Code,321,NAICS_2012_Code,321114, +BEA_2012_Summary_Code,321,NAICS_2012_Code,321211, +BEA_2012_Summary_Code,321,NAICS_2012_Code,321212, +BEA_2012_Summary_Code,321,NAICS_2012_Code,321213, +BEA_2012_Summary_Code,321,NAICS_2012_Code,321214, +BEA_2012_Summary_Code,321,NAICS_2012_Code,321219, +BEA_2012_Summary_Code,321,NAICS_2012_Code,321911, +BEA_2012_Summary_Code,321,NAICS_2012_Code,321912, +BEA_2012_Summary_Code,321,NAICS_2012_Code,321918, +BEA_2012_Summary_Code,321,NAICS_2012_Code,321920, +BEA_2012_Summary_Code,321,NAICS_2012_Code,321991, +BEA_2012_Summary_Code,321,NAICS_2012_Code,321992, +BEA_2012_Summary_Code,321,NAICS_2012_Code,321999, +BEA_2012_Summary_Code,322,NAICS_2012_Code,322110, +BEA_2012_Summary_Code,322,NAICS_2012_Code,322121, +BEA_2012_Summary_Code,322,NAICS_2012_Code,322122, +BEA_2012_Summary_Code,322,NAICS_2012_Code,322130, +BEA_2012_Summary_Code,322,NAICS_2012_Code,322211, +BEA_2012_Summary_Code,322,NAICS_2012_Code,322212, +BEA_2012_Summary_Code,322,NAICS_2012_Code,322219, +BEA_2012_Summary_Code,322,NAICS_2012_Code,322220, +BEA_2012_Summary_Code,322,NAICS_2012_Code,322230, +BEA_2012_Summary_Code,322,NAICS_2012_Code,322291, +BEA_2012_Summary_Code,322,NAICS_2012_Code,322299, +BEA_2012_Summary_Code,323,NAICS_2012_Code,323111, +BEA_2012_Summary_Code,323,NAICS_2012_Code,323113, +BEA_2012_Summary_Code,323,NAICS_2012_Code,323117, +BEA_2012_Summary_Code,323,NAICS_2012_Code,323120, +BEA_2012_Summary_Code,324,NAICS_2012_Code,324110, +BEA_2012_Summary_Code,324,NAICS_2012_Code,324121, +BEA_2012_Summary_Code,324,NAICS_2012_Code,324122, +BEA_2012_Summary_Code,324,NAICS_2012_Code,324191, +BEA_2012_Summary_Code,324,NAICS_2012_Code,324199, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325110, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325120, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325130, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325180, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325193, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325194, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325199, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325211, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325212, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325220, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325311, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325312, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325314, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325320, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325411, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325412, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325413, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325414, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325510, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325520, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325611, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325612, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325613, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325620, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325910, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325920, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325991, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325992, +BEA_2012_Summary_Code,325,NAICS_2012_Code,325998, +BEA_2012_Summary_Code,326,NAICS_2012_Code,326111, +BEA_2012_Summary_Code,326,NAICS_2012_Code,326112, +BEA_2012_Summary_Code,326,NAICS_2012_Code,326113, +BEA_2012_Summary_Code,326,NAICS_2012_Code,326121, +BEA_2012_Summary_Code,326,NAICS_2012_Code,326122, +BEA_2012_Summary_Code,326,NAICS_2012_Code,326130, +BEA_2012_Summary_Code,326,NAICS_2012_Code,326140, +BEA_2012_Summary_Code,326,NAICS_2012_Code,326150, +BEA_2012_Summary_Code,326,NAICS_2012_Code,326160, +BEA_2012_Summary_Code,326,NAICS_2012_Code,326191, +BEA_2012_Summary_Code,326,NAICS_2012_Code,326199, +BEA_2012_Summary_Code,326,NAICS_2012_Code,326211, +BEA_2012_Summary_Code,326,NAICS_2012_Code,326212, +BEA_2012_Summary_Code,326,NAICS_2012_Code,326220, +BEA_2012_Summary_Code,326,NAICS_2012_Code,326291, +BEA_2012_Summary_Code,326,NAICS_2012_Code,326299, +BEA_2012_Summary_Code,327,NAICS_2012_Code,327110, +BEA_2012_Summary_Code,327,NAICS_2012_Code,327120, +BEA_2012_Summary_Code,327,NAICS_2012_Code,327211, +BEA_2012_Summary_Code,327,NAICS_2012_Code,327212, +BEA_2012_Summary_Code,327,NAICS_2012_Code,327213, +BEA_2012_Summary_Code,327,NAICS_2012_Code,327215, +BEA_2012_Summary_Code,327,NAICS_2012_Code,327310, +BEA_2012_Summary_Code,327,NAICS_2012_Code,327320, +BEA_2012_Summary_Code,327,NAICS_2012_Code,327331, +BEA_2012_Summary_Code,327,NAICS_2012_Code,327332, +BEA_2012_Summary_Code,327,NAICS_2012_Code,327390, +BEA_2012_Summary_Code,327,NAICS_2012_Code,327410, +BEA_2012_Summary_Code,327,NAICS_2012_Code,327420, +BEA_2012_Summary_Code,327,NAICS_2012_Code,327910, +BEA_2012_Summary_Code,327,NAICS_2012_Code,327991, +BEA_2012_Summary_Code,327,NAICS_2012_Code,327992, +BEA_2012_Summary_Code,327,NAICS_2012_Code,327993, +BEA_2012_Summary_Code,327,NAICS_2012_Code,327999, +BEA_2012_Summary_Code,331,NAICS_2012_Code,331110, +BEA_2012_Summary_Code,331,NAICS_2012_Code,331210, +BEA_2012_Summary_Code,331,NAICS_2012_Code,331221, +BEA_2012_Summary_Code,331,NAICS_2012_Code,331222, +BEA_2012_Summary_Code,331,NAICS_2012_Code,331313, +BEA_2012_Summary_Code,331,NAICS_2012_Code,331314, +BEA_2012_Summary_Code,331,NAICS_2012_Code,331315, +BEA_2012_Summary_Code,331,NAICS_2012_Code,331318, +BEA_2012_Summary_Code,331,NAICS_2012_Code,331410, +BEA_2012_Summary_Code,331,NAICS_2012_Code,331420, +BEA_2012_Summary_Code,331,NAICS_2012_Code,331491, +BEA_2012_Summary_Code,331,NAICS_2012_Code,331492, +BEA_2012_Summary_Code,331,NAICS_2012_Code,331511, +BEA_2012_Summary_Code,331,NAICS_2012_Code,331512, +BEA_2012_Summary_Code,331,NAICS_2012_Code,331513, +BEA_2012_Summary_Code,331,NAICS_2012_Code,331523, +BEA_2012_Summary_Code,331,NAICS_2012_Code,331524, +BEA_2012_Summary_Code,331,NAICS_2012_Code,331529, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332111, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332112, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332114, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332117, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332119, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332215, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332216, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332311, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332312, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332313, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332321, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332322, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332323, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332410, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332420, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332431, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332439, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332510, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332613, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332618, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332710, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332721, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332722, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332811, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332812, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332813, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332911, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332912, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332913, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332919, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332991, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332992, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332993, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332994, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332996, +BEA_2012_Summary_Code,332,NAICS_2012_Code,332999, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333111, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333112, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333120, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333131, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333132, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333241, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333242, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333243, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333244, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333249, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333314, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333316, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333318, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333413, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333414, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333415, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333511, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333514, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333515, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333517, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333519, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333611, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333612, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333613, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333618, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333911, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333912, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333913, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333921, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333922, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333923, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333924, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333991, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333992, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333993, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333994, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333995, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333996, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333997, +BEA_2012_Summary_Code,333,NAICS_2012_Code,333999, +BEA_2012_Summary_Code,334,NAICS_2012_Code,334111, +BEA_2012_Summary_Code,334,NAICS_2012_Code,334112, +BEA_2012_Summary_Code,334,NAICS_2012_Code,334118, +BEA_2012_Summary_Code,334,NAICS_2012_Code,334210, +BEA_2012_Summary_Code,334,NAICS_2012_Code,334220, +BEA_2012_Summary_Code,334,NAICS_2012_Code,334290, +BEA_2012_Summary_Code,334,NAICS_2012_Code,334310, +BEA_2012_Summary_Code,334,NAICS_2012_Code,334412, +BEA_2012_Summary_Code,334,NAICS_2012_Code,334413, +BEA_2012_Summary_Code,334,NAICS_2012_Code,334416, +BEA_2012_Summary_Code,334,NAICS_2012_Code,334417, +BEA_2012_Summary_Code,334,NAICS_2012_Code,334418, +BEA_2012_Summary_Code,334,NAICS_2012_Code,334419, +BEA_2012_Summary_Code,334,NAICS_2012_Code,334510, +BEA_2012_Summary_Code,334,NAICS_2012_Code,334511, +BEA_2012_Summary_Code,334,NAICS_2012_Code,334512, +BEA_2012_Summary_Code,334,NAICS_2012_Code,334513, +BEA_2012_Summary_Code,334,NAICS_2012_Code,334514, +BEA_2012_Summary_Code,334,NAICS_2012_Code,334515, +BEA_2012_Summary_Code,334,NAICS_2012_Code,334516, +BEA_2012_Summary_Code,334,NAICS_2012_Code,334517, +BEA_2012_Summary_Code,334,NAICS_2012_Code,334519, +BEA_2012_Summary_Code,334,NAICS_2012_Code,334613, +BEA_2012_Summary_Code,334,NAICS_2012_Code,334614, +BEA_2012_Summary_Code,335,NAICS_2012_Code,335110, +BEA_2012_Summary_Code,335,NAICS_2012_Code,335121, +BEA_2012_Summary_Code,335,NAICS_2012_Code,335122, +BEA_2012_Summary_Code,335,NAICS_2012_Code,335129, +BEA_2012_Summary_Code,335,NAICS_2012_Code,335210, +BEA_2012_Summary_Code,335,NAICS_2012_Code,335221, +BEA_2012_Summary_Code,335,NAICS_2012_Code,335222, +BEA_2012_Summary_Code,335,NAICS_2012_Code,335224, +BEA_2012_Summary_Code,335,NAICS_2012_Code,335228, +BEA_2012_Summary_Code,335,NAICS_2012_Code,335311, +BEA_2012_Summary_Code,335,NAICS_2012_Code,335312, +BEA_2012_Summary_Code,335,NAICS_2012_Code,335313, +BEA_2012_Summary_Code,335,NAICS_2012_Code,335314, +BEA_2012_Summary_Code,335,NAICS_2012_Code,335911, +BEA_2012_Summary_Code,335,NAICS_2012_Code,335912, +BEA_2012_Summary_Code,335,NAICS_2012_Code,335921, +BEA_2012_Summary_Code,335,NAICS_2012_Code,335929, +BEA_2012_Summary_Code,335,NAICS_2012_Code,335931, +BEA_2012_Summary_Code,335,NAICS_2012_Code,335932, +BEA_2012_Summary_Code,335,NAICS_2012_Code,335991, +BEA_2012_Summary_Code,335,NAICS_2012_Code,335999, +BEA_2012_Summary_Code,3361MV,NAICS_2012_Code,336111, +BEA_2012_Summary_Code,3361MV,NAICS_2012_Code,336112, +BEA_2012_Summary_Code,3361MV,NAICS_2012_Code,336120, +BEA_2012_Summary_Code,3361MV,NAICS_2012_Code,336211, +BEA_2012_Summary_Code,3361MV,NAICS_2012_Code,336212, +BEA_2012_Summary_Code,3361MV,NAICS_2012_Code,336213, +BEA_2012_Summary_Code,3361MV,NAICS_2012_Code,336214, +BEA_2012_Summary_Code,3361MV,NAICS_2012_Code,336310, +BEA_2012_Summary_Code,3361MV,NAICS_2012_Code,336320, +BEA_2012_Summary_Code,3361MV,NAICS_2012_Code,336330, +BEA_2012_Summary_Code,3361MV,NAICS_2012_Code,336340, +BEA_2012_Summary_Code,3361MV,NAICS_2012_Code,336350, +BEA_2012_Summary_Code,3361MV,NAICS_2012_Code,336360, +BEA_2012_Summary_Code,3361MV,NAICS_2012_Code,336370, +BEA_2012_Summary_Code,3361MV,NAICS_2012_Code,336390, +BEA_2012_Summary_Code,3364OT,NAICS_2012_Code,336411, +BEA_2012_Summary_Code,3364OT,NAICS_2012_Code,336412, +BEA_2012_Summary_Code,3364OT,NAICS_2012_Code,336413, +BEA_2012_Summary_Code,3364OT,NAICS_2012_Code,336414, +BEA_2012_Summary_Code,3364OT,NAICS_2012_Code,336415, +BEA_2012_Summary_Code,3364OT,NAICS_2012_Code,336419, +BEA_2012_Summary_Code,3364OT,NAICS_2012_Code,336510, +BEA_2012_Summary_Code,3364OT,NAICS_2012_Code,336611, +BEA_2012_Summary_Code,3364OT,NAICS_2012_Code,336612, +BEA_2012_Summary_Code,3364OT,NAICS_2012_Code,336991, +BEA_2012_Summary_Code,3364OT,NAICS_2012_Code,336992, +BEA_2012_Summary_Code,3364OT,NAICS_2012_Code,336999, +BEA_2012_Summary_Code,337,NAICS_2012_Code,337110, +BEA_2012_Summary_Code,337,NAICS_2012_Code,337121, +BEA_2012_Summary_Code,337,NAICS_2012_Code,337122, +BEA_2012_Summary_Code,337,NAICS_2012_Code,337124, +BEA_2012_Summary_Code,337,NAICS_2012_Code,337125, +BEA_2012_Summary_Code,337,NAICS_2012_Code,337127, +BEA_2012_Summary_Code,337,NAICS_2012_Code,337211, +BEA_2012_Summary_Code,337,NAICS_2012_Code,337212, +BEA_2012_Summary_Code,337,NAICS_2012_Code,337214, +BEA_2012_Summary_Code,337,NAICS_2012_Code,337215, +BEA_2012_Summary_Code,337,NAICS_2012_Code,337910, +BEA_2012_Summary_Code,337,NAICS_2012_Code,337920, +BEA_2012_Summary_Code,339,NAICS_2012_Code,339112, +BEA_2012_Summary_Code,339,NAICS_2012_Code,339113, +BEA_2012_Summary_Code,339,NAICS_2012_Code,339114, +BEA_2012_Summary_Code,339,NAICS_2012_Code,339115, +BEA_2012_Summary_Code,339,NAICS_2012_Code,339116, +BEA_2012_Summary_Code,339,NAICS_2012_Code,339910, +BEA_2012_Summary_Code,339,NAICS_2012_Code,339920, +BEA_2012_Summary_Code,339,NAICS_2012_Code,339930, +BEA_2012_Summary_Code,339,NAICS_2012_Code,339940, +BEA_2012_Summary_Code,339,NAICS_2012_Code,339950, +BEA_2012_Summary_Code,339,NAICS_2012_Code,339991, +BEA_2012_Summary_Code,339,NAICS_2012_Code,339992, +BEA_2012_Summary_Code,339,NAICS_2012_Code,339993, +BEA_2012_Summary_Code,339,NAICS_2012_Code,339994, +BEA_2012_Summary_Code,339,NAICS_2012_Code,339995, +BEA_2012_Summary_Code,339,NAICS_2012_Code,339999, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423110, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423120, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423130, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423140, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423210, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423220, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423310, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423320, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423330, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423390, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423410, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423420, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423430, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423440, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423450, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423460, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423490, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423510, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423520, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423610, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423620, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423690, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423710, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423720, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423730, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423740, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423810, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423820, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423830, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423840, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423850, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423860, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423910, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423920, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423930, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423940, +BEA_2012_Summary_Code,42,NAICS_2012_Code,423990, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424110, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424120, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424130, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424210, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424310, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424320, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424330, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424340, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424410, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424420, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424430, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424440, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424450, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424460, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424470, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424480, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424490, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424510, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424520, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424590, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424610, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424690, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424710, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424720, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424810, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424820, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424910, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424920, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424930, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424940, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424950, +BEA_2012_Summary_Code,42,NAICS_2012_Code,424990, +BEA_2012_Summary_Code,42,NAICS_2012_Code,425110, +BEA_2012_Summary_Code,42,NAICS_2012_Code,425120, +BEA_2012_Summary_Code,441,NAICS_2012_Code,441110, +BEA_2012_Summary_Code,441,NAICS_2012_Code,441120, +BEA_2012_Summary_Code,441,NAICS_2012_Code,441210, +BEA_2012_Summary_Code,441,NAICS_2012_Code,441222, +BEA_2012_Summary_Code,441,NAICS_2012_Code,441228, +BEA_2012_Summary_Code,441,NAICS_2012_Code,441310, +BEA_2012_Summary_Code,441,NAICS_2012_Code,441320, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,442110, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,442210, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,442291, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,442299, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,443141, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,443142, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,444110, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,444120, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,444130, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,444190, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,444210, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,444220, +BEA_2012_Summary_Code,445,NAICS_2012_Code,445110, +BEA_2012_Summary_Code,445,NAICS_2012_Code,445120, +BEA_2012_Summary_Code,445,NAICS_2012_Code,445210, +BEA_2012_Summary_Code,445,NAICS_2012_Code,445220, +BEA_2012_Summary_Code,445,NAICS_2012_Code,445230, +BEA_2012_Summary_Code,445,NAICS_2012_Code,445291, +BEA_2012_Summary_Code,445,NAICS_2012_Code,445292, +BEA_2012_Summary_Code,445,NAICS_2012_Code,445299, +BEA_2012_Summary_Code,445,NAICS_2012_Code,445310, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,446110, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,446120, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,446130, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,446191, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,446199, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,447110, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,447190, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,448110, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,448120, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,448130, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,448140, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,448150, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,448190, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,448210, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,448310, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,448320, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,451110, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,451120, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,451130, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,451140, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,451211, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,451212, +BEA_2012_Summary_Code,452,NAICS_2012_Code,452111, +BEA_2012_Summary_Code,452,NAICS_2012_Code,452112, +BEA_2012_Summary_Code,452,NAICS_2012_Code,452910, +BEA_2012_Summary_Code,452,NAICS_2012_Code,452990, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,453110, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,453210, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,453220, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,453310, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,453910, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,453920, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,453930, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,453991, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,453998, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,454111, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,454112, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,454113, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,454210, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,454310, +BEA_2012_Summary_Code,4A0,NAICS_2012_Code,454390, +BEA_2012_Summary_Code,481,NAICS_2012_Code,481111, +BEA_2012_Summary_Code,481,NAICS_2012_Code,481112, +BEA_2012_Summary_Code,481,NAICS_2012_Code,481211, +BEA_2012_Summary_Code,481,NAICS_2012_Code,481212, +BEA_2012_Summary_Code,481,NAICS_2012_Code,481219, +BEA_2012_Summary_Code,482,NAICS_2012_Code,482111, +BEA_2012_Summary_Code,482,NAICS_2012_Code,482112, +BEA_2012_Summary_Code,483,NAICS_2012_Code,483111, +BEA_2012_Summary_Code,483,NAICS_2012_Code,483112, +BEA_2012_Summary_Code,483,NAICS_2012_Code,483113, +BEA_2012_Summary_Code,483,NAICS_2012_Code,483114, +BEA_2012_Summary_Code,483,NAICS_2012_Code,483211, +BEA_2012_Summary_Code,483,NAICS_2012_Code,483212, +BEA_2012_Summary_Code,484,NAICS_2012_Code,484110, +BEA_2012_Summary_Code,484,NAICS_2012_Code,484121, +BEA_2012_Summary_Code,484,NAICS_2012_Code,484122, +BEA_2012_Summary_Code,484,NAICS_2012_Code,484210, +BEA_2012_Summary_Code,484,NAICS_2012_Code,484220, +BEA_2012_Summary_Code,484,NAICS_2012_Code,484230, +BEA_2012_Summary_Code,485,NAICS_2012_Code,485111, +BEA_2012_Summary_Code,485,NAICS_2012_Code,485112, +BEA_2012_Summary_Code,485,NAICS_2012_Code,485113, +BEA_2012_Summary_Code,485,NAICS_2012_Code,485119, +BEA_2012_Summary_Code,485,NAICS_2012_Code,485210, +BEA_2012_Summary_Code,485,NAICS_2012_Code,485310, +BEA_2012_Summary_Code,485,NAICS_2012_Code,485320, +BEA_2012_Summary_Code,485,NAICS_2012_Code,485410, +BEA_2012_Summary_Code,485,NAICS_2012_Code,485510, +BEA_2012_Summary_Code,485,NAICS_2012_Code,485991, +BEA_2012_Summary_Code,485,NAICS_2012_Code,485999, +BEA_2012_Summary_Code,486,NAICS_2012_Code,486110, +BEA_2012_Summary_Code,486,NAICS_2012_Code,486210, +BEA_2012_Summary_Code,486,NAICS_2012_Code,486910, +BEA_2012_Summary_Code,486,NAICS_2012_Code,486990, +BEA_2012_Summary_Code,487OS,NAICS_2012_Code,487110, +BEA_2012_Summary_Code,487OS,NAICS_2012_Code,487210, +BEA_2012_Summary_Code,487OS,NAICS_2012_Code,487990, +BEA_2012_Summary_Code,487OS,NAICS_2012_Code,488111, +BEA_2012_Summary_Code,487OS,NAICS_2012_Code,488119, +BEA_2012_Summary_Code,487OS,NAICS_2012_Code,488190, +BEA_2012_Summary_Code,487OS,NAICS_2012_Code,488210, +BEA_2012_Summary_Code,487OS,NAICS_2012_Code,488310, +BEA_2012_Summary_Code,487OS,NAICS_2012_Code,488320, +BEA_2012_Summary_Code,487OS,NAICS_2012_Code,488330, +BEA_2012_Summary_Code,487OS,NAICS_2012_Code,488390, +BEA_2012_Summary_Code,487OS,NAICS_2012_Code,488410, +BEA_2012_Summary_Code,487OS,NAICS_2012_Code,488490, +BEA_2012_Summary_Code,487OS,NAICS_2012_Code,488510, +BEA_2012_Summary_Code,487OS,NAICS_2012_Code,488991, +BEA_2012_Summary_Code,487OS,NAICS_2012_Code,488999, +BEA_2012_Summary_Code,GFE,NAICS_2012_Code,491110, +BEA_2012_Summary_Code,487OS,NAICS_2012_Code,492110, +BEA_2012_Summary_Code,487OS,NAICS_2012_Code,492210, +BEA_2012_Summary_Code,493,NAICS_2012_Code,493110, +BEA_2012_Summary_Code,493,NAICS_2012_Code,493120, +BEA_2012_Summary_Code,493,NAICS_2012_Code,493130, +BEA_2012_Summary_Code,493,NAICS_2012_Code,493190, +BEA_2012_Summary_Code,511,NAICS_2012_Code,511110, +BEA_2012_Summary_Code,511,NAICS_2012_Code,511120, +BEA_2012_Summary_Code,511,NAICS_2012_Code,511130, +BEA_2012_Summary_Code,511,NAICS_2012_Code,511140, +BEA_2012_Summary_Code,511,NAICS_2012_Code,511191, +BEA_2012_Summary_Code,511,NAICS_2012_Code,511199, +BEA_2012_Summary_Code,511,NAICS_2012_Code,511210, +BEA_2012_Summary_Code,512,NAICS_2012_Code,512110, +BEA_2012_Summary_Code,512,NAICS_2012_Code,512120, +BEA_2012_Summary_Code,512,NAICS_2012_Code,512131, +BEA_2012_Summary_Code,512,NAICS_2012_Code,512132, +BEA_2012_Summary_Code,512,NAICS_2012_Code,512191, +BEA_2012_Summary_Code,512,NAICS_2012_Code,512199, +BEA_2012_Summary_Code,512,NAICS_2012_Code,512210, +BEA_2012_Summary_Code,512,NAICS_2012_Code,512220, +BEA_2012_Summary_Code,512,NAICS_2012_Code,512230, +BEA_2012_Summary_Code,512,NAICS_2012_Code,512240, +BEA_2012_Summary_Code,512,NAICS_2012_Code,512290, +BEA_2012_Summary_Code,513,NAICS_2012_Code,515111, +BEA_2012_Summary_Code,513,NAICS_2012_Code,515112, +BEA_2012_Summary_Code,513,NAICS_2012_Code,515120, +BEA_2012_Summary_Code,513,NAICS_2012_Code,515210, +BEA_2012_Summary_Code,513,NAICS_2012_Code,517110, +BEA_2012_Summary_Code,513,NAICS_2012_Code,517210, +BEA_2012_Summary_Code,513,NAICS_2012_Code,517410, +BEA_2012_Summary_Code,513,NAICS_2012_Code,517911, +BEA_2012_Summary_Code,513,NAICS_2012_Code,517919, +BEA_2012_Summary_Code,514,NAICS_2012_Code,518210, +BEA_2012_Summary_Code,514,NAICS_2012_Code,519110, +BEA_2012_Summary_Code,514,NAICS_2012_Code,519120, +BEA_2012_Summary_Code,514,NAICS_2012_Code,519130, +BEA_2012_Summary_Code,514,NAICS_2012_Code,519190, +BEA_2012_Summary_Code,521CI,NAICS_2012_Code,521110, +BEA_2012_Summary_Code,521CI,NAICS_2012_Code,522110, +BEA_2012_Summary_Code,521CI,NAICS_2012_Code,522120, +BEA_2012_Summary_Code,521CI,NAICS_2012_Code,522130, +BEA_2012_Summary_Code,521CI,NAICS_2012_Code,522190, +BEA_2012_Summary_Code,521CI,NAICS_2012_Code,522210, +BEA_2012_Summary_Code,521CI,NAICS_2012_Code,522220, +BEA_2012_Summary_Code,521CI,NAICS_2012_Code,522291, +BEA_2012_Summary_Code,521CI,NAICS_2012_Code,522292, +BEA_2012_Summary_Code,521CI,NAICS_2012_Code,522293, +BEA_2012_Summary_Code,521CI,NAICS_2012_Code,522294, +BEA_2012_Summary_Code,521CI,NAICS_2012_Code,522298, +BEA_2012_Summary_Code,521CI,NAICS_2012_Code,522310, +BEA_2012_Summary_Code,521CI,NAICS_2012_Code,522320, +BEA_2012_Summary_Code,521CI,NAICS_2012_Code,522390, +BEA_2012_Summary_Code,523,NAICS_2012_Code,523110, +BEA_2012_Summary_Code,523,NAICS_2012_Code,523120, +BEA_2012_Summary_Code,523,NAICS_2012_Code,523130, +BEA_2012_Summary_Code,523,NAICS_2012_Code,523140, +BEA_2012_Summary_Code,523,NAICS_2012_Code,523210, +BEA_2012_Summary_Code,523,NAICS_2012_Code,523910, +BEA_2012_Summary_Code,523,NAICS_2012_Code,523920, +BEA_2012_Summary_Code,523,NAICS_2012_Code,523930, +BEA_2012_Summary_Code,523,NAICS_2012_Code,523991, +BEA_2012_Summary_Code,523,NAICS_2012_Code,523999, +BEA_2012_Summary_Code,524,NAICS_2012_Code,524113, +BEA_2012_Summary_Code,524,NAICS_2012_Code,524114, +BEA_2012_Summary_Code,524,NAICS_2012_Code,524126, +BEA_2012_Summary_Code,524,NAICS_2012_Code,524127, +BEA_2012_Summary_Code,524,NAICS_2012_Code,524128, +BEA_2012_Summary_Code,524,NAICS_2012_Code,524130, +BEA_2012_Summary_Code,524,NAICS_2012_Code,524210, +BEA_2012_Summary_Code,524,NAICS_2012_Code,524291, +BEA_2012_Summary_Code,524,NAICS_2012_Code,524292, +BEA_2012_Summary_Code,524,NAICS_2012_Code,524298, +BEA_2012_Summary_Code,525,NAICS_2012_Code,525110, +BEA_2012_Summary_Code,525,NAICS_2012_Code,525120, +BEA_2012_Summary_Code,525,NAICS_2012_Code,525190, +BEA_2012_Summary_Code,525,NAICS_2012_Code,525910, +BEA_2012_Summary_Code,525,NAICS_2012_Code,525920, +BEA_2012_Summary_Code,525,NAICS_2012_Code,525990, +BEA_2012_Summary_Code,HS,NAICS_2012_Code,531110, +BEA_2012_Summary_Code,ORE,NAICS_2012_Code,531110, +BEA_2012_Summary_Code,HS,NAICS_2012_Code,531120, +BEA_2012_Summary_Code,ORE,NAICS_2012_Code,531120, +BEA_2012_Summary_Code,HS,NAICS_2012_Code,531130, +BEA_2012_Summary_Code,ORE,NAICS_2012_Code,531130, +BEA_2012_Summary_Code,HS,NAICS_2012_Code,531190, +BEA_2012_Summary_Code,ORE,NAICS_2012_Code,531190, +BEA_2012_Summary_Code,HS,NAICS_2012_Code,531210, +BEA_2012_Summary_Code,ORE,NAICS_2012_Code,531210, +BEA_2012_Summary_Code,ORE,NAICS_2012_Code,531311, +BEA_2012_Summary_Code,HS,NAICS_2012_Code,531311, +BEA_2012_Summary_Code,HS,NAICS_2012_Code,531312, +BEA_2012_Summary_Code,ORE,NAICS_2012_Code,531312, +BEA_2012_Summary_Code,HS,NAICS_2012_Code,531320, +BEA_2012_Summary_Code,ORE,NAICS_2012_Code,531320, +BEA_2012_Summary_Code,HS,NAICS_2012_Code,531390, +BEA_2012_Summary_Code,ORE,NAICS_2012_Code,531390, +BEA_2012_Summary_Code,532RL,NAICS_2012_Code,532111, +BEA_2012_Summary_Code,HS,NAICS_2012_Code,532111, +BEA_2012_Summary_Code,HS,NAICS_2012_Code,532112, +BEA_2012_Summary_Code,532RL,NAICS_2012_Code,532112, +BEA_2012_Summary_Code,HS,NAICS_2012_Code,532120, +BEA_2012_Summary_Code,532RL,NAICS_2012_Code,532120, +BEA_2012_Summary_Code,HS,NAICS_2012_Code,532210, +BEA_2012_Summary_Code,532RL,NAICS_2012_Code,532210, +BEA_2012_Summary_Code,HS,NAICS_2012_Code,532220, +BEA_2012_Summary_Code,532RL,NAICS_2012_Code,532220, +BEA_2012_Summary_Code,532RL,NAICS_2012_Code,532230, +BEA_2012_Summary_Code,HS,NAICS_2012_Code,532230, +BEA_2012_Summary_Code,HS,NAICS_2012_Code,532291, +BEA_2012_Summary_Code,532RL,NAICS_2012_Code,532291, +BEA_2012_Summary_Code,HS,NAICS_2012_Code,532292, +BEA_2012_Summary_Code,532RL,NAICS_2012_Code,532292, +BEA_2012_Summary_Code,HS,NAICS_2012_Code,532299, +BEA_2012_Summary_Code,532RL,NAICS_2012_Code,532299, +BEA_2012_Summary_Code,532RL,NAICS_2012_Code,532310, +BEA_2012_Summary_Code,HS,NAICS_2012_Code,532310, +BEA_2012_Summary_Code,HS,NAICS_2012_Code,532411, +BEA_2012_Summary_Code,532RL,NAICS_2012_Code,532411, +BEA_2012_Summary_Code,HS,NAICS_2012_Code,532412, +BEA_2012_Summary_Code,532RL,NAICS_2012_Code,532412, +BEA_2012_Summary_Code,HS,NAICS_2012_Code,532420, +BEA_2012_Summary_Code,532RL,NAICS_2012_Code,532420, +BEA_2012_Summary_Code,HS,NAICS_2012_Code,532490, +BEA_2012_Summary_Code,532RL,NAICS_2012_Code,532490, +BEA_2012_Summary_Code,532RL,NAICS_2012_Code,533110, +BEA_2012_Summary_Code,HS,NAICS_2012_Code,533110, +BEA_2012_Summary_Code,5411,NAICS_2012_Code,541110, +BEA_2012_Summary_Code,5411,NAICS_2012_Code,541120, +BEA_2012_Summary_Code,5411,NAICS_2012_Code,541191, +BEA_2012_Summary_Code,5411,NAICS_2012_Code,541199, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541211, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541213, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541214, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541219, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541310, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541320, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541330, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541340, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541350, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541360, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541370, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541380, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541410, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541420, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541430, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541490, +BEA_2012_Summary_Code,5415,NAICS_2012_Code,541511, +BEA_2012_Summary_Code,5415,NAICS_2012_Code,541512, +BEA_2012_Summary_Code,5415,NAICS_2012_Code,541513, +BEA_2012_Summary_Code,5415,NAICS_2012_Code,541519, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541611, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541612, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541613, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541614, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541618, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541620, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541690, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541711, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541712, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541720, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541810, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541820, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541830, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541840, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541850, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541860, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541870, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541890, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541910, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541921, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541922, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541930, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541940, +BEA_2012_Summary_Code,5412OP,NAICS_2012_Code,541990, +BEA_2012_Summary_Code,55,NAICS_2012_Code,551111, +BEA_2012_Summary_Code,55,NAICS_2012_Code,551112, +BEA_2012_Summary_Code,55,NAICS_2012_Code,551114, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561110, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561210, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561311, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561312, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561320, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561330, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561410, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561421, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561422, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561431, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561439, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561440, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561450, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561491, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561492, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561499, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561510, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561520, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561591, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561599, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561611, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561612, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561613, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561621, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561622, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561710, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561720, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561730, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561740, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561790, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561910, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561920, +BEA_2012_Summary_Code,561,NAICS_2012_Code,561990, +BEA_2012_Summary_Code,562,NAICS_2012_Code,562111, +BEA_2012_Summary_Code,562,NAICS_2012_Code,562112, +BEA_2012_Summary_Code,562,NAICS_2012_Code,562119, +BEA_2012_Summary_Code,562,NAICS_2012_Code,562211, +BEA_2012_Summary_Code,562,NAICS_2012_Code,562212, +BEA_2012_Summary_Code,562,NAICS_2012_Code,562213, +BEA_2012_Summary_Code,562,NAICS_2012_Code,562219, +BEA_2012_Summary_Code,562,NAICS_2012_Code,562910, +BEA_2012_Summary_Code,562,NAICS_2012_Code,562920, +BEA_2012_Summary_Code,562,NAICS_2012_Code,562991, +BEA_2012_Summary_Code,562,NAICS_2012_Code,562998, +BEA_2012_Summary_Code,61,NAICS_2012_Code,611110, +BEA_2012_Summary_Code,61,NAICS_2012_Code,611210, +BEA_2012_Summary_Code,61,NAICS_2012_Code,611310, +BEA_2012_Summary_Code,61,NAICS_2012_Code,611410, +BEA_2012_Summary_Code,61,NAICS_2012_Code,611420, +BEA_2012_Summary_Code,61,NAICS_2012_Code,611430, +BEA_2012_Summary_Code,61,NAICS_2012_Code,611511, +BEA_2012_Summary_Code,61,NAICS_2012_Code,611512, +BEA_2012_Summary_Code,61,NAICS_2012_Code,611513, +BEA_2012_Summary_Code,61,NAICS_2012_Code,611519, +BEA_2012_Summary_Code,61,NAICS_2012_Code,611610, +BEA_2012_Summary_Code,61,NAICS_2012_Code,611620, +BEA_2012_Summary_Code,61,NAICS_2012_Code,611630, +BEA_2012_Summary_Code,61,NAICS_2012_Code,611691, +BEA_2012_Summary_Code,61,NAICS_2012_Code,611692, +BEA_2012_Summary_Code,61,NAICS_2012_Code,611699, +BEA_2012_Summary_Code,61,NAICS_2012_Code,611710, +BEA_2012_Summary_Code,621,NAICS_2012_Code,621111, +BEA_2012_Summary_Code,621,NAICS_2012_Code,621112, +BEA_2012_Summary_Code,621,NAICS_2012_Code,621210, +BEA_2012_Summary_Code,621,NAICS_2012_Code,621310, +BEA_2012_Summary_Code,621,NAICS_2012_Code,621320, +BEA_2012_Summary_Code,621,NAICS_2012_Code,621330, +BEA_2012_Summary_Code,621,NAICS_2012_Code,621340, +BEA_2012_Summary_Code,621,NAICS_2012_Code,621391, +BEA_2012_Summary_Code,621,NAICS_2012_Code,621399, +BEA_2012_Summary_Code,621,NAICS_2012_Code,621410, +BEA_2012_Summary_Code,621,NAICS_2012_Code,621420, +BEA_2012_Summary_Code,621,NAICS_2012_Code,621491, +BEA_2012_Summary_Code,621,NAICS_2012_Code,621492, +BEA_2012_Summary_Code,621,NAICS_2012_Code,621493, +BEA_2012_Summary_Code,621,NAICS_2012_Code,621498, +BEA_2012_Summary_Code,621,NAICS_2012_Code,621511, +BEA_2012_Summary_Code,621,NAICS_2012_Code,621512, +BEA_2012_Summary_Code,621,NAICS_2012_Code,621610, +BEA_2012_Summary_Code,621,NAICS_2012_Code,621910, +BEA_2012_Summary_Code,621,NAICS_2012_Code,621991, +BEA_2012_Summary_Code,621,NAICS_2012_Code,621999, +BEA_2012_Summary_Code,622,NAICS_2012_Code,622110, +BEA_2012_Summary_Code,622,NAICS_2012_Code,622210, +BEA_2012_Summary_Code,622,NAICS_2012_Code,622310, +BEA_2012_Summary_Code,623,NAICS_2012_Code,623110, +BEA_2012_Summary_Code,623,NAICS_2012_Code,623210, +BEA_2012_Summary_Code,623,NAICS_2012_Code,623220, +BEA_2012_Summary_Code,623,NAICS_2012_Code,623311, +BEA_2012_Summary_Code,623,NAICS_2012_Code,623312, +BEA_2012_Summary_Code,623,NAICS_2012_Code,623990, +BEA_2012_Summary_Code,624,NAICS_2012_Code,624110, +BEA_2012_Summary_Code,624,NAICS_2012_Code,624120, +BEA_2012_Summary_Code,624,NAICS_2012_Code,624190, +BEA_2012_Summary_Code,624,NAICS_2012_Code,624210, +BEA_2012_Summary_Code,624,NAICS_2012_Code,624221, +BEA_2012_Summary_Code,624,NAICS_2012_Code,624229, +BEA_2012_Summary_Code,624,NAICS_2012_Code,624230, +BEA_2012_Summary_Code,624,NAICS_2012_Code,624310, +BEA_2012_Summary_Code,624,NAICS_2012_Code,624410, +BEA_2012_Summary_Code,711AS,NAICS_2012_Code,711110, +BEA_2012_Summary_Code,711AS,NAICS_2012_Code,711120, +BEA_2012_Summary_Code,711AS,NAICS_2012_Code,711130, +BEA_2012_Summary_Code,711AS,NAICS_2012_Code,711190, +BEA_2012_Summary_Code,711AS,NAICS_2012_Code,711211, +BEA_2012_Summary_Code,711AS,NAICS_2012_Code,711212, +BEA_2012_Summary_Code,711AS,NAICS_2012_Code,711219, +BEA_2012_Summary_Code,711AS,NAICS_2012_Code,711310, +BEA_2012_Summary_Code,711AS,NAICS_2012_Code,711320, +BEA_2012_Summary_Code,711AS,NAICS_2012_Code,711410, +BEA_2012_Summary_Code,711AS,NAICS_2012_Code,711510, +BEA_2012_Summary_Code,711AS,NAICS_2012_Code,712110, +BEA_2012_Summary_Code,711AS,NAICS_2012_Code,712120, +BEA_2012_Summary_Code,711AS,NAICS_2012_Code,712130, +BEA_2012_Summary_Code,711AS,NAICS_2012_Code,712190, +BEA_2012_Summary_Code,713,NAICS_2012_Code,713110, +BEA_2012_Summary_Code,713,NAICS_2012_Code,713120, +BEA_2012_Summary_Code,713,NAICS_2012_Code,713210, +BEA_2012_Summary_Code,713,NAICS_2012_Code,713290, +BEA_2012_Summary_Code,713,NAICS_2012_Code,713910, +BEA_2012_Summary_Code,713,NAICS_2012_Code,713920, +BEA_2012_Summary_Code,713,NAICS_2012_Code,713930, +BEA_2012_Summary_Code,713,NAICS_2012_Code,713940, +BEA_2012_Summary_Code,713,NAICS_2012_Code,713950, +BEA_2012_Summary_Code,713,NAICS_2012_Code,713990, +BEA_2012_Summary_Code,721,NAICS_2012_Code,721110, +BEA_2012_Summary_Code,721,NAICS_2012_Code,721120, +BEA_2012_Summary_Code,721,NAICS_2012_Code,721191, +BEA_2012_Summary_Code,721,NAICS_2012_Code,721199, +BEA_2012_Summary_Code,721,NAICS_2012_Code,721211, +BEA_2012_Summary_Code,721,NAICS_2012_Code,721214, +BEA_2012_Summary_Code,721,NAICS_2012_Code,721310, +BEA_2012_Summary_Code,722,NAICS_2012_Code,722310, +BEA_2012_Summary_Code,722,NAICS_2012_Code,722320, +BEA_2012_Summary_Code,722,NAICS_2012_Code,722330, +BEA_2012_Summary_Code,722,NAICS_2012_Code,722410, +BEA_2012_Summary_Code,722,NAICS_2012_Code,722511, +BEA_2012_Summary_Code,722,NAICS_2012_Code,722513, +BEA_2012_Summary_Code,722,NAICS_2012_Code,722514, +BEA_2012_Summary_Code,722,NAICS_2012_Code,722515, +BEA_2012_Summary_Code,81,NAICS_2012_Code,811111, +BEA_2012_Summary_Code,81,NAICS_2012_Code,811112, +BEA_2012_Summary_Code,81,NAICS_2012_Code,811113, +BEA_2012_Summary_Code,81,NAICS_2012_Code,811118, +BEA_2012_Summary_Code,81,NAICS_2012_Code,811121, +BEA_2012_Summary_Code,81,NAICS_2012_Code,811122, +BEA_2012_Summary_Code,81,NAICS_2012_Code,811191, +BEA_2012_Summary_Code,81,NAICS_2012_Code,811192, +BEA_2012_Summary_Code,81,NAICS_2012_Code,811198, +BEA_2012_Summary_Code,81,NAICS_2012_Code,811211, +BEA_2012_Summary_Code,81,NAICS_2012_Code,811212, +BEA_2012_Summary_Code,81,NAICS_2012_Code,811213, +BEA_2012_Summary_Code,81,NAICS_2012_Code,811219, +BEA_2012_Summary_Code,81,NAICS_2012_Code,811310, +BEA_2012_Summary_Code,81,NAICS_2012_Code,811411, +BEA_2012_Summary_Code,81,NAICS_2012_Code,811412, +BEA_2012_Summary_Code,81,NAICS_2012_Code,811420, +BEA_2012_Summary_Code,81,NAICS_2012_Code,811430, +BEA_2012_Summary_Code,81,NAICS_2012_Code,811490, +BEA_2012_Summary_Code,81,NAICS_2012_Code,812111, +BEA_2012_Summary_Code,81,NAICS_2012_Code,812112, +BEA_2012_Summary_Code,81,NAICS_2012_Code,812113, +BEA_2012_Summary_Code,81,NAICS_2012_Code,812191, +BEA_2012_Summary_Code,81,NAICS_2012_Code,812199, +BEA_2012_Summary_Code,81,NAICS_2012_Code,812210, +BEA_2012_Summary_Code,81,NAICS_2012_Code,812220, +BEA_2012_Summary_Code,81,NAICS_2012_Code,812310, +BEA_2012_Summary_Code,81,NAICS_2012_Code,812320, +BEA_2012_Summary_Code,81,NAICS_2012_Code,812331, +BEA_2012_Summary_Code,81,NAICS_2012_Code,812332, +BEA_2012_Summary_Code,81,NAICS_2012_Code,812910, +BEA_2012_Summary_Code,81,NAICS_2012_Code,812921, +BEA_2012_Summary_Code,81,NAICS_2012_Code,812922, +BEA_2012_Summary_Code,81,NAICS_2012_Code,812930, +BEA_2012_Summary_Code,81,NAICS_2012_Code,812990, +BEA_2012_Summary_Code,81,NAICS_2012_Code,813110, +BEA_2012_Summary_Code,81,NAICS_2012_Code,813211, +BEA_2012_Summary_Code,81,NAICS_2012_Code,813212, +BEA_2012_Summary_Code,81,NAICS_2012_Code,813219, +BEA_2012_Summary_Code,81,NAICS_2012_Code,813311, +BEA_2012_Summary_Code,81,NAICS_2012_Code,813312, +BEA_2012_Summary_Code,81,NAICS_2012_Code,813319, +BEA_2012_Summary_Code,81,NAICS_2012_Code,813410, +BEA_2012_Summary_Code,81,NAICS_2012_Code,813910, +BEA_2012_Summary_Code,81,NAICS_2012_Code,813920, +BEA_2012_Summary_Code,81,NAICS_2012_Code,813930, +BEA_2012_Summary_Code,81,NAICS_2012_Code,813940, +BEA_2012_Summary_Code,81,NAICS_2012_Code,813990, +BEA_2012_Summary_Code,81,NAICS_2012_Code,814110, +BEA_2012_Summary_Code,GSLG,NAICS_2012_Code,921110, +BEA_2012_Summary_Code,GFGN,NAICS_2012_Code,921110, +BEA_2012_Summary_Code,GSLG,NAICS_2012_Code,921120, +BEA_2012_Summary_Code,GFGN,NAICS_2012_Code,921120, +BEA_2012_Summary_Code,GFGN,NAICS_2012_Code,921130, +BEA_2012_Summary_Code,GSLG,NAICS_2012_Code,921130, +BEA_2012_Summary_Code,GSLG,NAICS_2012_Code,921140, +BEA_2012_Summary_Code,GFGN,NAICS_2012_Code,921140, +BEA_2012_Summary_Code,GSLG,NAICS_2012_Code,921150, +BEA_2012_Summary_Code,GSLG,NAICS_2012_Code,921190, +BEA_2012_Summary_Code,GFGN,NAICS_2012_Code,921190, +BEA_2012_Summary_Code,GSLG,NAICS_2012_Code,922110, +BEA_2012_Summary_Code,GFGN,NAICS_2012_Code,922110, +BEA_2012_Summary_Code,GSLG,NAICS_2012_Code,922120, +BEA_2012_Summary_Code,GFGN,NAICS_2012_Code,922120, +BEA_2012_Summary_Code,GSLG,NAICS_2012_Code,922130, +BEA_2012_Summary_Code,GFGN,NAICS_2012_Code,922130, +BEA_2012_Summary_Code,GSLG,NAICS_2012_Code,922140, +BEA_2012_Summary_Code,GFGN,NAICS_2012_Code,922140, +BEA_2012_Summary_Code,GSLG,NAICS_2012_Code,922150, +BEA_2012_Summary_Code,GFGN,NAICS_2012_Code,922150, +BEA_2012_Summary_Code,GFGN,NAICS_2012_Code,922160, +BEA_2012_Summary_Code,GSLG,NAICS_2012_Code,922160, +BEA_2012_Summary_Code,GSLG,NAICS_2012_Code,922190, +BEA_2012_Summary_Code,GFGN,NAICS_2012_Code,922190, +BEA_2012_Summary_Code,GSLG,NAICS_2012_Code,923110, +BEA_2012_Summary_Code,GFGN,NAICS_2012_Code,923110, +BEA_2012_Summary_Code,GSLG,NAICS_2012_Code,923120, +BEA_2012_Summary_Code,GFGN,NAICS_2012_Code,923120, +BEA_2012_Summary_Code,GSLG,NAICS_2012_Code,923130, +BEA_2012_Summary_Code,GFGN,NAICS_2012_Code,923130, +BEA_2012_Summary_Code,GFGN,NAICS_2012_Code,923140, +BEA_2012_Summary_Code,GSLG,NAICS_2012_Code,924110, +BEA_2012_Summary_Code,GFGN,NAICS_2012_Code,924110, +BEA_2012_Summary_Code,GSLG,NAICS_2012_Code,924120, +BEA_2012_Summary_Code,GFGN,NAICS_2012_Code,924120, +BEA_2012_Summary_Code,GSLG,NAICS_2012_Code,925110, +BEA_2012_Summary_Code,GFGN,NAICS_2012_Code,925110, +BEA_2012_Summary_Code,GFGN,NAICS_2012_Code,925120, +BEA_2012_Summary_Code,GSLG,NAICS_2012_Code,925120, +BEA_2012_Summary_Code,GSLG,NAICS_2012_Code,926110, +BEA_2012_Summary_Code,GFGN,NAICS_2012_Code,926110, +BEA_2012_Summary_Code,GSLG,NAICS_2012_Code,926120, +BEA_2012_Summary_Code,GSLG,NAICS_2012_Code,926130, +BEA_2012_Summary_Code,GFGN,NAICS_2012_Code,926130, +BEA_2012_Summary_Code,GSLG,NAICS_2012_Code,926140, +BEA_2012_Summary_Code,GFGN,NAICS_2012_Code,926140, +BEA_2012_Summary_Code,GSLG,NAICS_2012_Code,926150, +BEA_2012_Summary_Code,GFGN,NAICS_2012_Code,926150, +BEA_2012_Summary_Code,GFE,NAICS_2012_Code,927110, +BEA_2012_Summary_Code,GFGD,NAICS_2012_Code,928110, +BEA_2012_Summary_Code,GFGN,NAICS_2012_Code,928120, +BEA_2012_Summary_Code,F010,NAICS_2012_Code,F01000, +BEA_2012_Summary_Code,GFE,NAICS_2012_Code,S00101, +BEA_2012_Summary_Code,GSLE,NAICS_2012_Code,S00201, +BEA_2012_Summary_Code,GSLE,NAICS_2012_Code,S00202, diff --git a/flowsa/data/source_catalog.yaml b/flowsa/data/source_catalog.yaml index bfe15eedb..a842d9be8 100644 --- a/flowsa/data/source_catalog.yaml +++ b/flowsa/data/source_catalog.yaml @@ -121,6 +121,18 @@ EPA_GHGI: sector-like_activities: False activity_schema: sector_aggregation_level: "aggregated" +EPA_StateGHGI: + class: + - Chemicals + sector-like_activities: False + activity_schema: + sector_aggregation_level: "aggregated" +EPA_SIT: + class: + - Chemicals + sector-like_activities: False + activity_schema: + sector_aggregation_level: "aggregated" EPA_NEI_Nonpoint: class: - Chemicals @@ -170,6 +182,12 @@ StatCan_LFS: sector-like_activities: False activity_schema: sector_aggregation_level: "aggregated" +stateio: + class: + - Money + sector-like_activities: False #update to true once alternate activity_schema in place + # activity_schema: BEA_2012_Summary_Code + sector_aggregation_level: "disaggregated" USDA_CoA_Cropland: class: - Land diff --git a/flowsa/data_source_scripts/BEA.py b/flowsa/data_source_scripts/BEA.py index d64cd2b5d..7add8178b 100644 --- a/flowsa/data_source_scripts/BEA.py +++ b/flowsa/data_source_scripts/BEA.py @@ -62,28 +62,8 @@ def bea_use_detail_br_parse(*, year, **_): f'_Detail_Use_PRO_BeforeRedef.csv' df_raw = pd.read_csv(csv_load) - # first column is the commodity being consumed - df = df_raw.rename(columns={'Unnamed: 0': 'ActivityProducedBy'}) - - # use "melt" fxn to convert colummns into rows - df = df.melt(id_vars=["ActivityProducedBy"], - var_name="ActivityConsumedBy", - value_name="FlowAmount") - - df['Year'] = str(year) - # hardcode data - df['FlowName'] = "USD" + str(year) - df["Class"] = "Money" - df["FlowType"] = "TECHNOSPHERE_FLOW" - df['Description'] = 'BEA_2012_Detail_Code' + df = bea_detail_parse(df_raw, year) df["SourceName"] = "BEA_Use_Detail_PRO_BeforeRedef" - df["Location"] = US_FIPS - df['LocationSystem'] = "FIPS_2015" - # original unit in million USD - df['FlowAmount'] = df['FlowAmount'] * 1000000 - df["Unit"] = "USD" - df['DataReliability'] = 5 # tmp - df['DataCollection'] = 5 # tmp return df @@ -96,10 +76,17 @@ def bea_make_detail_br_parse(*, year, **_): flowbyactivity specifications """ # Read directly into a pandas df - df_raw = pd.read_csv(externaldatapath + "BEA_" + str(year) + - "_Detail_Make_BeforeRedef.csv") + csv_load = f'{externaldatapath}BEA_{str(year)}' \ + f'_Detail_Make_BeforeRedef.csv' + df_raw = pd.read_csv(csv_load) + + df = bea_detail_parse(df_raw, year) + df["SourceName"] = "BEA_Make_Detail_BeforeRedef" - # first column is the industry + return df + + +def bea_detail_parse(df_raw, year): df = df_raw.rename(columns={'Unnamed: 0': 'ActivityProducedBy'}) # use "melt" fxn to convert colummns into rows @@ -109,11 +96,10 @@ def bea_make_detail_br_parse(*, year, **_): df['Year'] = str(year) # hardcode data - df['FlowName'] = "USD" + str(year) + df['FlowName'] = f"USD{str(year)}" df["Class"] = "Money" df["FlowType"] = "TECHNOSPHERE_FLOW" df['Description'] = 'BEA_2012_Detail_Code' - df["SourceName"] = "BEA_Make_Detail_BeforeRedef" df["Location"] = US_FIPS df['LocationSystem'] = "FIPS_2015" # original unit in million USD @@ -121,7 +107,6 @@ def bea_make_detail_br_parse(*, year, **_): df["Unit"] = "USD" df['DataReliability'] = 5 # tmp df['DataCollection'] = 5 # tmp - return df @@ -187,9 +172,16 @@ def subset_and_allocate_BEA_table(df, attr, **_): """ Temporary function to mimic use of 2nd helper allocation dataset """ + df = subset_BEA_table(df, attr) v = {'geoscale_to_use': 'national'} method2 = {'target_sector_source': 'NAICS_2012_Code'} + + import importlib + fxn = getattr(importlib.import_module( + 'flowsa.data_source_scripts.BLS_QCEW'), + "bls_clean_allocation_fba_w_sec") + attr2 = {"helper_source": "BLS_QCEW", "helper_method": "proportional", "helper_source_class": "Employment", @@ -200,7 +192,7 @@ def subset_and_allocate_BEA_table(df, attr, **_): "Number of employees, Private"], "helper_from_scale": "national", "allocation_from_scale": "national", - "clean_helper_fba_wsec": "bls_clean_allocation_fba_w_sec"} + "clean_helper_fba_wsec": fxn} df2 = allocation_helper(df, attr2, method2, v, False) # Drop remaining rows with no sectors e.g. T001 and other final demands df2 = df2.dropna(subset=['SectorConsumedBy']).reset_index(drop=True) diff --git a/flowsa/data_source_scripts/BLS_QCEW.py b/flowsa/data_source_scripts/BLS_QCEW.py index e85c3367a..aa7aa7b3f 100644 --- a/flowsa/data_source_scripts/BLS_QCEW.py +++ b/flowsa/data_source_scripts/BLS_QCEW.py @@ -136,21 +136,21 @@ def bls_qcew_parse(*, df_list, year, **_): return df2 -def clean_bls_qcew_fba_for_employment_sat_table(fba_df, **kwargs): +def clean_bls_qcew_fba_for_employment_sat_table(fba, **_): """ When creating the employment satellite table for use in useeior, - modify the flow name to match prior methodology for mapping/impact factors + modify the flow name to match prior methodology for mapping/impact factors. + clean_fba_df_fxn - :param fba_df: df, flowbyactivity - :param kwargs: dictionary, can include attr, a dictionary of parameters - in the FBA method yaml + :param fba: df, flowbyactivity :return: df, flowbyactivity, with modified flow names """ + # rename flowname value for c in ['FlowName', 'Flowable']: - fba_df[c] = fba_df[c].str.replace('Number of employees', 'Jobs') + fba[c] = fba[c].str.replace('Number of employees', 'Jobs') - return fba_df + return fba def bls_clean_allocation_fba_w_sec(df_w_sec, **kwargs): diff --git a/flowsa/data_source_scripts/CalRecycle_WasteCharacterization.py b/flowsa/data_source_scripts/CalRecycle_WasteCharacterization.py index 8228217b9..fa4f73f84 100644 --- a/flowsa/data_source_scripts/CalRecycle_WasteCharacterization.py +++ b/flowsa/data_source_scripts/CalRecycle_WasteCharacterization.py @@ -19,7 +19,7 @@ from flowsa.settings import externaldatapath from flowsa.sectormapping import get_fba_allocation_subset, \ add_sectors_to_flowbyactivity -from flowsa.dataclean import replace_strings_with_NoneType +from flowsa.dataclean import replace_strings_with_NoneType, standardize_units def produced_by(entry): @@ -109,16 +109,17 @@ def calR_parse(*, year, **_): return output -def keep_generated_quantity(fba, **kwargs): +def keep_generated_quantity(fba, **_): """ Function to clean CalRecycles FBA to remove quantities not assigned as Generated :param fba: df, FBA format - :param kwargs: dictionary, can include attr, a dictionary of parameters in - the FBA method yaml :return: df, modified CalRecycles FBA """ - fba = fba[fba['Description'] == 'Generated'] + fba = fba[fba['Description'] == 'Generated'].reset_index(drop=True) + # if no mapping performed, still update units + if 'tons' in fba['Unit'].values: + fba = standardize_units(fba) return fba @@ -146,6 +147,10 @@ def apply_tons_per_employee_per_year_to_states(fbs, method, **_): # Calculate tons per employee per year per material and sector in CA bls_CA = bls[bls['Location'] == '06000'] # California + # aggregate all employment prior to generating tpepy + bls_CA = (bls_CA.groupby(['Location','Year','SectorProducedBy']) + .agg({'Employees':'sum'}) + .reset_index()) tpepy = fbs.merge(bls_CA, how='inner') tpepy['TPEPY'] = np.divide(tpepy['FlowAmount'], tpepy['Employees'], out=np.zeros_like(tpepy['Employees']), diff --git a/flowsa/data_source_scripts/Census_CBP.py b/flowsa/data_source_scripts/Census_CBP.py index c805a76ab..16f06f052 100644 --- a/flowsa/data_source_scripts/Census_CBP.py +++ b/flowsa/data_source_scripts/Census_CBP.py @@ -34,11 +34,7 @@ def Census_CBP_URL_helper(*, build_url, year, **_): # This is only for years 2010 and 2011. This is done because the State # query that gets all counties returns too many results and errors out. if year in ['2010', '2011']: - if year == '2011': - fips_year = '2010' - else: - fips_year = '2010' - county_fips_df = get_county_FIPS(fips_year) + county_fips_df = get_county_FIPS('2010') county_fips = county_fips_df.FIPS for d in county_fips: url = build_url @@ -82,16 +78,15 @@ def Census_CBP_URL_helper(*, build_url, year, **_): urls_census.append(url) else: FIPS_2 = get_all_state_FIPS_2()['FIPS_2'] - for c in FIPS_2: + for state in FIPS_2: url = build_url - url = url.replace("__stateFIPS__", c) + url = url.replace("__stateFIPS__", state) # specified NAICS code year depends on year of data - if year in ['2017']: + if year in ['2017', '2018', '2019', '2020']: url = url.replace("__NAICS__", "NAICS2017") - url = url.replace("__countyFIPS__", "*") - if year in ['2012', '2013', '2014', '2015', '2016']: + elif year in ['2012', '2013', '2014', '2015', '2016']: url = url.replace("__NAICS__", "NAICS2012") - url = url.replace("__countyFIPS__", "*") + url = url.replace("__countyFIPS__", "*") urls_census.append(url) return urls_census @@ -152,6 +147,10 @@ def census_cbp_parse(*, df_list, year, **_): value_name="FlowAmount") # specify unit based on flowname df['Unit'] = np.where(df["FlowName"] == 'Annual payroll', "USD", "p") + # Payroll in units of thousand USD + df['FlowAmount'] = np.where(df["FlowName"] == 'Annual payroll', + df['FlowAmount'] * 1000, + df['FlowAmount']) # specify class df.loc[df['FlowName'] == 'Number of employees', 'Class'] = 'Employment' df.loc[df['FlowName'] == 'Number of establishments', 'Class'] = 'Other' diff --git a/flowsa/data_source_scripts/EIA_CBECS_Land.py b/flowsa/data_source_scripts/EIA_CBECS_Land.py index 610d28389..dba2a5536 100644 --- a/flowsa/data_source_scripts/EIA_CBECS_Land.py +++ b/flowsa/data_source_scripts/EIA_CBECS_Land.py @@ -233,15 +233,15 @@ def standardize_eia_cbecs_land_activity_names(df, column_to_standardize): return df -def cbecs_land_fba_cleanup(fba_load): +def cbecs_land_fba_cleanup(fba, **_): """ Clean up the land fba for use in allocation - :param fba_load: df, eia cbecs land flowbyactivity format + :param fba: df, eia cbecs land flowbyactivity format :return: df, flowbyactivity with modified values """ # estimate floor space using number of floors - fba = calculate_floorspace_based_on_number_of_floors(fba_load) + fba = calculate_floorspace_based_on_number_of_floors(fba) # calculate the land area in addition to building footprint fba1 = calculate_total_facility_land_area(fba) diff --git a/flowsa/data_source_scripts/EIA_MECS.py b/flowsa/data_source_scripts/EIA_MECS.py index 7e9f6f790..40086e9d2 100644 --- a/flowsa/data_source_scripts/EIA_MECS.py +++ b/flowsa/data_source_scripts/EIA_MECS.py @@ -436,7 +436,7 @@ def eia_mecs_energy_clean_allocation_fba_w_sec( return df2 -def mecs_land_fba_cleanup(fba): +def mecs_land_fba_cleanup(fba, **_): """ Modify the EIA MECS Land FBA :param fba: df, EIA MECS Land FBA format @@ -452,7 +452,7 @@ def mecs_land_fba_cleanup(fba): return fba -def mecs_land_fba_cleanup_for_land_2012_fbs(fba): +def mecs_land_fba_cleanup_for_land_2012_fbs(fba, **_): """ The 'land_national_2012' FlowBySector uses MECS 2014 data, set MECS year to 2012 @@ -460,7 +460,7 @@ def mecs_land_fba_cleanup_for_land_2012_fbs(fba): :return: df, EIA MECS Land FBA modified """ - fba = mecs_land_fba_cleanup(fba) + fba = mecs_land_fba_cleanup(fba=fba) # reset the EIA MECS Land year from 2014 to 2012 to match # the USDA ERS MLU year diff --git a/flowsa/data_source_scripts/EPA_CDDPath.py b/flowsa/data_source_scripts/EPA_CDDPath.py index 33d2339bc..8ef581457 100644 --- a/flowsa/data_source_scripts/EPA_CDDPath.py +++ b/flowsa/data_source_scripts/EPA_CDDPath.py @@ -14,6 +14,7 @@ from flowsa.location import US_FIPS from flowsa.settings import externaldatapath from flowsa.flowbyfunctions import assign_fips_location_system +from flowsa.dataclean import standardize_units # Read pdf into list of DataFrame @@ -95,17 +96,21 @@ def combine_cdd_path(*, resp, **_): return df -def assign_wood_to_engineering(df): +def assign_wood_to_engineering(fba, **_): """clean_fba_df_fxn that reclassifies Wood from 'Other' to 'Other - Wood' so that its mapping can be adjusted to only use 237990/Heavy engineering NAICS according to method in Meyer et al. 2020 - :param df: df, FBA of CDDPath + :param fba: df, FBA of CDDPath :return: df, CDDPath FBA with wood reassigned """ # Update wood to a new activity for improved mapping - df.loc[((df.FlowName == 'Wood') & - (df.ActivityProducedBy == 'Other')), + fba.loc[((fba.FlowName == 'Wood') & + (fba.ActivityProducedBy == 'Other')), 'ActivityProducedBy'] = 'Other - Wood' - return df + # if no mapping performed, still update units + if 'short tons' in fba['Unit'].values: + fba = standardize_units(fba) + + return fba diff --git a/flowsa/data_source_scripts/EPA_GHGI.py b/flowsa/data_source_scripts/EPA_GHGI.py index 3d6e5c0b5..ab02079f4 100644 --- a/flowsa/data_source_scripts/EPA_GHGI.py +++ b/flowsa/data_source_scripts/EPA_GHGI.py @@ -15,8 +15,10 @@ from flowsa.dataclean import replace_NoneType_with_empty_cells from flowsa.settings import log, externaldatapath from flowsa.schema import flow_by_activity_fields +from flowsa.common import load_yaml_dict from flowsa.data_source_scripts import EIA_MECS + SECTOR_DICT = {'Res.': 'Residential', 'Comm.': 'Commercial', 'Ind.': 'Industrial', @@ -635,18 +637,35 @@ def get_manufacturing_energy_ratios(year): 'Natural Gas': 'Natural Gas', } - # TODO make this year dynamic + def closest_value(input_list, input_value): + difference = lambda input_list : abs(input_list - input_value) + return min(input_list, key=difference) + + mecs_year = closest_value(load_yaml_dict('EIA_MECS_Energy', + flowbytype='FBA').get('years'), + year) + # Filter MECS for total national energy consumption for manufacturing sectors mecs = load_fba_w_standardized_units(datasource='EIA_MECS_Energy', - year=year, + year=mecs_year, flowclass='Energy') mecs = mecs.loc[(mecs['ActivityConsumedBy'] == '31-33') & (mecs['Location'] == '00000')].reset_index(drop=True) mecs = EIA_MECS.mecs_energy_fba_cleanup(mecs, None) - # TODO dynamically change the table imported here based on year - ghgi = load_fba_w_standardized_units(datasource='EPA_GHGI_T_A_14', - year=2016, + # Identify the GHGI table that matches EIA_MECS + for t, v in (load_yaml_dict('EPA_GHGI', 'FBA') + .get('Annex').get('Annex').items()): + if ((v.get('class') == 'Energy') + & ('Energy Consumption Data' in v.get('desc')) + & (v.get('year') == str(mecs_year))): + table = f"EPA_GHGI_T_{t.replace('-', '_')}" + break + else: + log.error('unable to identify corresponding GHGI table') + + ghgi = load_fba_w_standardized_units(datasource=table, + year=mecs_year, flowclass='Energy') ghgi = ghgi[ghgi['ActivityConsumedBy']=='Industrial'].reset_index(drop=True) @@ -661,7 +680,7 @@ def get_manufacturing_energy_ratios(year): return pct_dict -def allocate_industrial_combustion(df): +def allocate_industrial_combustion(fba, source_dict, **_): """ Split industrial combustion emissions into two buckets to be further allocated. @@ -669,9 +688,7 @@ def allocate_industrial_combustion(df): EIA MECS relative to EPA GHGI. Create new activities to distinguish those which use EIA MECS as allocation source and those that use alternate source. """ - # TODO make this year dynamic - year = 2014 - pct_dict = get_manufacturing_energy_ratios(year) + pct_dict = get_manufacturing_energy_ratios(source_dict.get('year')) # activities reflect flows in A_14 and 3_8 and 3_9 activities_to_split = {'Industrial Other Coal Industrial': 'Coal', @@ -680,29 +697,30 @@ def allocate_industrial_combustion(df): 'Natural gas industrial': 'Natural Gas'} for activity, fuel in activities_to_split.items(): - df_subset = df.loc[df['ActivityProducedBy'] == activity].reset_index(drop=True) + df_subset = fba.loc[fba['ActivityProducedBy'] == activity].reset_index(drop=True) if len(df_subset) == 0: continue df_subset['FlowAmount'] = df_subset['FlowAmount'] * pct_dict[fuel] df_subset['ActivityProducedBy'] = f"{activity} - Manufacturing" - df.loc[df['ActivityProducedBy'] == activity, - 'FlowAmount'] = df['FlowAmount'] * (1-pct_dict[fuel]) - df = pd.concat([df, df_subset], ignore_index=True) + fba.loc[fba['ActivityProducedBy'] == activity, + 'FlowAmount'] = fba['FlowAmount'] * (1-pct_dict[fuel]) + fba = pd.concat([fba, df_subset], ignore_index=True) - return df + return fba -def split_HFCs_by_type(df): - """Speciates HFCs and PFCs for all activities based on T_4_99.""" +def split_HFCs_by_type(fba, **_): + """Speciates HFCs and PFCs for all activities based on T_4_99. + clean_fba_before_mapping_df_fxn""" splits = load_fba_w_standardized_units(datasource='EPA_GHGI_T_4_99', - year=df['Year'][0]) + year=fba['Year'][0]) splits['pct'] = splits['FlowAmount'] / splits['FlowAmount'].sum() splits = splits[['FlowName', 'pct']] - speciated_df = df.apply(lambda x: [p * x['FlowAmount'] for p in splits['pct']], + speciated_df = fba.apply(lambda x: [p * x['FlowAmount'] for p in splits['pct']], axis=1, result_type='expand') speciated_df.columns = splits['FlowName'] - speciated_df = pd.concat([df, speciated_df], axis=1) + speciated_df = pd.concat([fba, speciated_df], axis=1) speciated_df = speciated_df.melt(id_vars=flow_by_activity_fields.keys(), var_name='Flow') speciated_df['FlowName'] = speciated_df['Flow'] @@ -782,20 +800,20 @@ def split_HFC_foams(df): return df -def clean_HFC_fba(df): +def clean_HFC_fba(fba, **_): """Adjust HFC emissions for improved parsing. clean_fba_before_mapping_df_fxn used in EPA_GHGI_T_4_101.""" - df = subtract_HFC_transport_emissions(df) + df = subtract_HFC_transport_emissions(fba) df = allocate_HFC_to_residential(df) df = split_HFC_foams(df) df = split_HFCs_by_type(df) return df -def remove_HFC_kt(df): +def remove_HFC_kt(fba, **_): """Remove records of emissions in kt, data are also provided in MMT CO2e. clean_fba_before_mapping_df_fxn used in EPA_GHGI_T_4_50.""" - return df.loc[df['Unit'] != 'kt'] + return fba.loc[fba['Unit'] != 'kt'] def adjust_transport_activities(df, **_): diff --git a/flowsa/data_source_scripts/EPA_NEI.py b/flowsa/data_source_scripts/EPA_NEI.py index a5e01edab..3fe4e99d4 100644 --- a/flowsa/data_source_scripts/EPA_NEI.py +++ b/flowsa/data_source_scripts/EPA_NEI.py @@ -155,7 +155,7 @@ def epa_nei_nonpoint_parse(*, df_list, source, year, config, **_): return df -def clean_NEI_fba(fba): +def clean_NEI_fba(fba, **_): """ Clean up the NEI FBA for use in FBS creation :param fba: df, FBA format @@ -181,14 +181,14 @@ def clean_NEI_fba(fba): return fba -def clean_NEI_fba_no_pesticides(fba): +def clean_NEI_fba_no_pesticides(fba, **_): """ Clean up the NEI FBA with no pesicides for use in FBS creation :param fba: df, FBA format :return: df, modified FBA """ fba = drop_pesticides(fba) - fba = clean_NEI_fba(fba) + fba = clean_NEI_fba(fba=fba) return fba @@ -208,7 +208,7 @@ def remove_duplicate_NEI_flows(df): return df -def drop_GHGs(df): +def drop_GHGs(df, *_): """ GHGs are included in some NEI datasets. If these data are not compiled together with GHGRP, need to remove them as they will be @@ -216,18 +216,14 @@ def drop_GHGs(df): :param df: df, FBA format :return: df """"" - # Flow names reflect source data prior to FEDEFL mapping, using 'FlowName' - # instead of 'Flowable' flowlist = [ - 'Carbon Dioxide', + 'Carbon Dioxide', 'Carbon dioxide', 'Methane', - 'Nitrous Oxide', - 'Sulfur Hexafluoride', + 'Nitrous Oxide', 'Nitrous oxide', + 'Sulfur Hexafluoride', 'Sulfur hexafluoride', ] - - df = df.loc[~df['FlowName'].isin(flowlist)] - - return df + flow_var = 'Flowable' if 'Flowable' in df.columns else 'FlowName' + return df.query(f'{flow_var} not in @flowlist') def drop_pesticides(df): diff --git a/flowsa/data_source_scripts/EPA_SIT.py b/flowsa/data_source_scripts/EPA_SIT.py new file mode 100644 index 000000000..969b60bbd --- /dev/null +++ b/flowsa/data_source_scripts/EPA_SIT.py @@ -0,0 +1,121 @@ +# EPA_SIT.py (flowsa) +# !/usr/bin/env python3 +# coding=utf-8 +""" +Loads EPA State Inventory Tool (SIT) data for state specified from external +data directory. Parses EPA SIT data to flowbyactivity format. +""" + +import pandas as pd +import os +from flowsa.settings import externaldatapath, log +from flowsa.flowbyfunctions import assign_fips_location_system +from flowsa.location import apply_county_FIPS + +def epa_sit_parse(*, source, year, config, **_): + + state = config['state'] + filepath = f"{externaldatapath}/SIT_data/{state}/{config['file']}" + # dictionary containing Excel sheet-specific information + sheet_dict = config['sheet_dict'] + # initialize the dataframe + df0 = pd.DataFrame() + + if not os.path.exists(filepath): + raise FileNotFoundError(f'SIT file not found in {filepath}') + + # for each sheet in the Excel file containing data... + for sheet, sheet_dict in config.get('sheet_dict').items(): + sheetname = sheet_dict.get('sheetname', sheet) + tablename = sheet_dict.get('tablename') + if tablename: + sheetandtable = f'{sheetname}, {tablename}' + else: + sheetandtable = sheetname + tablename = sheet_dict.get('tablename', sheetname) + log.debug(f'Loading data from: {sheetname}...') + # read in data from Excel sheet + df = pd.read_excel(filepath, + sheet_name = sheetname, + header=sheet_dict.get('header', 2), + skiprows=range(sheet_dict.get('skiprowstart', 0), + sheet_dict.get('skiprowend', 0)), + usecols="B:AG", + nrows=sheet_dict.get('nrows')) + df.columns = df.columns.map(str) + df['ActivityProducedBy'] = df.iloc[:,0] + + # for each row in the data table... + # ...emissions categories will be renamed with the format + # 'sheet name, emissions category' + # ...emissions subcategories will be renamed with the format + # 'sheet name, emissions category, emissions subcategory' + for ind in df.index: + current_header = df['ActivityProducedBy'][ind].strip() + # for level 1 headers... + if current_header in sheet_dict.get('headers'): + active_header = current_header + if sheet_dict.get('subgroup') == 'activitybyflow': + df.loc[ind, 'FlowName'] = active_header + elif sheet_dict.get('subgroup') == 'flow': + df.loc[ind, 'FlowName'] = 'Total N2O and CH4 Emissions' + df.loc[ind,'ActivityProducedBy'] = ( + f'{sheetandtable}, {active_header}') + # for level 2 headers... + elif current_header not in sheet_dict.get('subsubheaders',''): + active_subheader = df['ActivityProducedBy'][ind].strip() + if sheet_dict.get('subgroup') == 'flow': + df.loc[ind, 'FlowName'] = active_subheader + df.loc[ind,'ActivityProducedBy'] = ( + f'{sheetandtable}, {active_header}') + elif sheet_dict.get('subgroup') == 'activitybyflow': + df.loc[ind, 'FlowName'] = active_header + df.loc[ind,'ActivityProducedBy'] = ( + f'{sheetandtable}, {active_subheader}') + else: + df.loc[ind,'ActivityProducedBy'] = ( + f'{sheetandtable}, {active_header}, ' + f'{active_subheader}') + # for level 3 headers (only occur in IndirectCO2 and Agriculture tabs)... + else: + subsubheader = df['ActivityProducedBy'][ind].strip() + df.loc[ind,'ActivityProducedBy'] = ( + f'{sheetandtable}, {active_header}, ' + f'{active_subheader}, {subsubheader}') + + # drop all columns except the desired emissions year and the + # emissions activity source + df = df.filter([year, 'ActivityProducedBy', 'FlowName']) + # rename columns + df = df.rename(columns={year: 'FlowAmount'}) + # add sheet-specific hardcoded data + if 'subgroup' not in sheet_dict: + df['FlowName'] = sheet_dict.get('flow') + df['Unit'] = sheet_dict.get('unit') + df['Description'] = sheetname + + # concatenate dataframe from each sheet with existing master dataframe + df0 = pd.concat([df0, df]) + + # add general hardcoded data + df0['Class'] = 'Chemicals' + df0['SourceName'] = source + df0['FlowType'] = "ELEMENTARY_FLOW" + df0['Compartment'] = 'air' + df0['Year'] = year + df0['DataReliability'] = 5 + df0['DataCollection'] = 5 + + # add state FIPS code + df0['State'] = state + df0['County'] = '' + df0 = apply_county_FIPS(df0, year='2015', source_state_abbrev=True) + # add FIPS location system + df0 = assign_fips_location_system(df0, '2015') + + return df0 + +if __name__ == '__main__': + import flowsa + flowsa.flowbyactivity.main(source='EPA_SIT', year='2017') + fba = flowsa.getFlowByActivity('EPA_SIT', '2017') diff --git a/flowsa/data_source_scripts/EPA_StateGHGI.py b/flowsa/data_source_scripts/EPA_StateGHGI.py new file mode 100644 index 000000000..8f59134f8 --- /dev/null +++ b/flowsa/data_source_scripts/EPA_StateGHGI.py @@ -0,0 +1,99 @@ +# EPA_StateGHGI.py (flowsa) +# !/usr/bin/env python3 +# coding=utf-8 +""" +Inventory of US GHGs from EPA disaggregated to States +""" +import json +import pandas as pd +from flowsa.settings import externaldatapath +from flowsa.location import apply_county_FIPS +from flowsa.flowbyfunctions import assign_fips_location_system +import flowsa.exceptions + + +def epa_state_ghgi_parse(*, source, year, config, **_): + + try: + with open(externaldatapath + config.get('file')) as f: + data = json.load(f) + except FileNotFoundError: + raise FileNotFoundError('State GHGI data not yet available for ' + 'external users') + + data_df = pd.DataFrame(data) + activity_cols = ['SECTOR', 'SOURCE', 'SUBSOURCE', 'FUEL_TYPE', + 'SUB_REFERENCE', 'SECSUB_REFERENCE'] + + states = data_df[['STATE']].drop_duplicates() + flows = data_df[['GHG_NAME']].drop_duplicates() + + df = data_df.melt(id_vars = activity_cols + ['STATE'] + ['GHG_NAME'], + value_vars=f'EMISSION_{year}', + var_name = 'Year', + value_name = 'FlowAmount') + df['Year'] = year + df['Unit'] = 'MMT CO2e' # TODO confirm units + df['FlowType'] = 'ELEMENTARY_FLOW' + df['SourceName'] = source + df['Class'] = 'Chemicals' + df['Compartment'] = 'air' + + df.rename(columns={'STATE': 'State', + 'GHG_NAME': 'FlowName'}, + inplace=True) + + df['ActivityProducedBy'] = (df[activity_cols] + .apply(lambda row: ' - '.join( + row.values.astype(str)), axis=1)) + df['ActivityProducedBy'] = (df['ActivityProducedBy'] + .str.replace(' - None', '')) + df.drop(columns=activity_cols, inplace=True) + activities = df[['ActivityProducedBy']].drop_duplicates() + + df['County'] = '' + df = apply_county_FIPS(df) + df = assign_fips_location_system(df, '2015') + df.drop(columns=['County'], inplace=True) + + return df + + +def remove_select_states(fba, source_dict, **_): + """ + clean_fba_df_fxn to remove selected states so they can be added + from alternate sources. State abbreviations must be passed as list + in method parameter 'state_list' + + :param fba: df + :param source_dict: dictionary of source methods includes 'state_list' + key of states to remove + """ + state_list = source_dict.get('state_list') + state_df = pd.DataFrame(state_list, columns=['State']) + state_df['County'] ='' + state_df = apply_county_FIPS(state_df) + df_subset = fba[~fba['Location'].isin(state_df['Location'])] + return df_subset + + +def tag_biogenic_activities(fba, source_dict, **_): + """ + clean_fba_before_mapping_df_fxn to tag emissions from passed activities + as biogenic. Activities passed as list in paramter 'activity_list'. + """ + a_list = source_dict.get('activity_list') + if a_list is None: + raise flowsa.exceptions.FBSMethodConstructionError( + message="Activities to tag must be passed in FBS parameter " + "'activity_list'") + fba.loc[fba['ActivityProducedBy'].isin(a_list), + 'FlowName'] = fba['FlowName'] + ' - biogenic' + + return fba + + +if __name__ == '__main__': + import flowsa + flowsa.flowbyactivity.main(source='EPA_StateGHGI', year='2017') + fba = flowsa.getFlowByActivity('EPA_StateGHGI', '2017') diff --git a/flowsa/data_source_scripts/USGS_NWIS_WU.py b/flowsa/data_source_scripts/USGS_NWIS_WU.py index 53f01defc..5fcf66f36 100644 --- a/flowsa/data_source_scripts/USGS_NWIS_WU.py +++ b/flowsa/data_source_scripts/USGS_NWIS_WU.py @@ -354,10 +354,10 @@ def standardize_usgs_nwis_names(flowbyactivity_df): return flowbyactivity_df -def usgs_fba_data_cleanup(df): +def usgs_fba_data_cleanup(fba, **_): """ Clean up the dataframe to prepare for flowbysector. Used in flowbysector.py - :param df: df, FBA format + :param fba: df, FBA format :return: df, modified FBA """ @@ -368,9 +368,9 @@ def usgs_fba_data_cleanup(df): vLogDetailed.info('Removing all rows for Commercial Data because does not ' 'exist for all states and causes issues as information ' 'on Public Supply deliveries.') - dfa = df[~df['Description'].str.lower().str.contains( + dfa = fba[~fba['Description'].str.lower().str.contains( 'commercial|closed-loop cooling|once-through')] - calculate_flowamount_diff_between_dfs(df, dfa) + calculate_flowamount_diff_between_dfs(fba, dfa) # calculated NET PUBLIC SUPPLY by subtracting out deliveries to domestic vLogDetailed.info('Modify the public supply values to generate ' 'NET public supply by subtracting out deliveries ' diff --git a/flowsa/data_source_scripts/stateio.py b/flowsa/data_source_scripts/stateio.py new file mode 100644 index 000000000..d3aa77771 --- /dev/null +++ b/flowsa/data_source_scripts/stateio.py @@ -0,0 +1,80 @@ +# stateio.py (flowsa) +# !/usr/bin/env python3 +# coding=utf-8 +""" +Supporting functions for accessing files from stateior via data commons. +https://github.com/USEPA/stateior +""" + +import os +import pandas as pd + +from esupy.processed_data_mgmt import download_from_remote, Paths,\ + load_preprocessed_output +from flowsa.metadata import set_fb_meta +from flowsa.location import us_state_abbrev, apply_county_FIPS +from flowsa.flowbyfunctions import assign_fips_location_system + + +def parse_statior(*, source, year, config, **_): + """parse_response_fxn for stateio make and use tables""" + # Prepare meta for downloading stateior datasets + name = config.get('datatype') + fname = f"{name}_{year}" + meta = set_fb_meta(fname, "") + meta.tool = 'stateio' + meta.ext = 'rds' + stateio_paths = Paths() + stateio_paths.local_path = os.path.realpath(stateio_paths.local_path + + "/stateio") + # Download and load the latest version from remote + download_from_remote(meta, stateio_paths) + states = load_preprocessed_output(meta, stateio_paths) + + data_dict = {} + + # uses rpy2 + # this .rds is stored as a list of named dataframes by state + for state in us_state_abbrev.keys(): + df = states.rx2(state) + df2 = df.melt(ignore_index=False, value_name = 'FlowAmount', + var_name = 'ActivityConsumedBy') + df2['ActivityProducedBy'] = df2.index + if source == 'stateio_Make_Summary': + # Adjust the index by removing the state: STATE.SECTOR + df2['ActivityProducedBy'] = df2[ + 'ActivityProducedBy'].str.split(".", expand=True)[1] + df2.reset_index(drop=True, inplace=True) + df2['State'] = state + data_dict[state] = df2 + + fba = pd.concat(data_dict, ignore_index=True) + fba.dropna(subset=['FlowAmount'], inplace=True) + + # Gross Output + if 'GO' in source and 'ActivityConsumedBy' in fba.columns: + fba = fba.drop(columns=['ActivityConsumedBy']) + + # Assign location + fba['County'] = '' + fba = apply_county_FIPS(fba) + fba = assign_fips_location_system(fba, '2015') + fba.drop(columns=['County'], inplace=True) + + # Hardcoded data + fba['Year'] = year + fba['SourceName'] = source + fba['Class'] = 'Money' + fba['Unit'] = "USD" + fba['FlowName'] = f"USD{year}" + fba["FlowType"] = "TECHNOSPHERE_FLOW" + fba['DataReliability'] = 5 # tmp + fba['DataCollection'] = 5 # tmp + return fba + + +if __name__ == "__main__": + import flowsa + source = 'stateio_Industry_GO' + flowsa.flowbyactivity.main(year=2017, source=source) + fba = flowsa.getFlowByActivity(source, 2017) diff --git a/flowsa/data_source_scripts/stewiFBS.py b/flowsa/data_source_scripts/stewiFBS.py index 284a1f110..77f0053c3 100644 --- a/flowsa/data_source_scripts/stewiFBS.py +++ b/flowsa/data_source_scripts/stewiFBS.py @@ -15,6 +15,7 @@ import os import pandas as pd from esupy.dqi import get_weighted_average +from esupy.processed_data_mgmt import read_source_metadata from flowsa.allocation import equally_allocate_parent_to_child_naics from flowsa.flowbyfunctions import assign_fips_location_system,\ aggregate_and_subset_for_target_sectors @@ -24,6 +25,12 @@ from flowsa.schema import flow_by_sector_fields from flowsa.settings import log, process_adjustmentpath from flowsa.validation import replace_naics_w_naics_from_another_year +import stewicombo +import stewi +from stewicombo.overlaphandler import remove_default_flow_overlaps +from stewicombo.globals import addChemicalMatches, compile_metadata,\ + set_stewicombo_meta +import facilitymatcher def stewicombo_to_sector(yaml_load, method, fbsconfigpath=None): @@ -43,12 +50,6 @@ def stewicombo_to_sector(yaml_load, method, fbsconfigpath=None): :param fbsconfigpath, str, optional path to an FBS method outside flowsa repo :return: df, FBS format """ - - import stewicombo - from flowsa.data_source_scripts.EPA_NEI import drop_GHGs - - # determine if fxns specified in FBS method yaml - functions = yaml_load.get('functions', []) inventory_name = yaml_load.get('local_inventory_name') df = None @@ -89,15 +90,8 @@ def stewicombo_to_sector(yaml_load, method, fbsconfigpath=None): df['MetaSources'] = df['Source'] - if 'drop_GHGs' in functions: - df = drop_GHGs(df) - functions.remove('drop_GHGs') - fbs = prepare_stewi_fbs(df, yaml_load, method) - for function in functions: - fbs = getattr(sys.modules[__name__], function)(fbs) - return fbs @@ -113,8 +107,6 @@ def stewi_to_sector(yaml_load, method, *_): :param method: dictionary, FBS method :return: df, FBS format """ - import stewi - # determine if fxns specified in FBS method yaml functions = yaml_load.get('functions', []) @@ -158,10 +150,6 @@ def reassign_process_to_sectors(df, year, file_list, fbsconfigpath): :param fbsconfigpath, str, optional path to an FBS method outside flowsa repo :return: df """ - import stewi - from stewicombo.overlaphandler import remove_default_flow_overlaps - from stewicombo.globals import addChemicalMatches - df_adj = pd.DataFrame() for file in file_list: fpath = f"{process_adjustmentpath}{file}.csv" @@ -229,7 +217,6 @@ def extract_facility_data(inventory_dict): {'NEI':'2017', 'TRI':'2017'}) :return: df """ - import stewi facilities_list = [] # load facility data from stewi output directory, keeping only the # facility IDs, and geographic information @@ -259,7 +246,6 @@ def obtain_NAICS_from_facility_matcher(inventory_list): :param inventory_list: a list of inventories (e.g., ['NEI', 'TRI']) :return: df """ - import facilitymatcher # Access NAICS From facility matcher and assign based on FRS_ID all_NAICS = \ facilitymatcher.get_FRS_NAICSInfo_for_facility_list( @@ -383,10 +369,15 @@ def add_stewi_metadata(inventory_dict): {'NEI':'2017', 'TRI':'2017'}) :return: combined dictionary of metadata from each inventory """ - from stewicombo.globals import compile_metadata return compile_metadata(inventory_dict) +def add_stewicombo_metadata(inventory_name): + """Access locally stored stewicombo metadata by filename""" + return read_source_metadata(stewicombo.globals.paths, + set_stewicombo_meta(inventory_name)) + + if __name__ == "__main__": import flowsa flowsa.flowbysector.main(method='CRHW_state_2017') diff --git a/flowsa/dataclean.py b/flowsa/dataclean.py index 477e55c6d..8a07161b6 100644 --- a/flowsa/dataclean.py +++ b/flowsa/dataclean.py @@ -210,13 +210,15 @@ def standardize_units(df): np.where(df['Unit'] == 'million Cubic metres/year', 'kg', df['Unit']) # Convert mass units (LB or TON) to kg - df.loc[:, 'FlowAmount'] = np.where(df['Unit'] == 'TON', + df.loc[:, 'FlowAmount'] = np.where(df['Unit'].isin(['TON', 'tons', + 'short tons']), df['FlowAmount'] * ton_to_kg, df['FlowAmount']) df.loc[:, 'FlowAmount'] = np.where(df['Unit'] == 'LB', df['FlowAmount'] * lb_to_kg, df['FlowAmount']) - df.loc[:, 'Unit'] = np.where(df['Unit'].isin(['TON', 'LB']), + df.loc[:, 'Unit'] = np.where(df['Unit'].isin(['TON', 'tons', + 'short tons', 'LB']), 'kg', df['Unit']) return df diff --git a/flowsa/exceptions.py b/flowsa/exceptions.py new file mode 100644 index 000000000..5d922fe92 --- /dev/null +++ b/flowsa/exceptions.py @@ -0,0 +1,46 @@ +# exceptions.py (flowsa) +# !/usr/bin/env python3 +# coding=utf-8 + +"""Defines custom exceptions for flowsa""" + + +class FBANotAvailableError(Exception): + def __init__(self, method=None, year=None, message=None): + if message is None: + message = ("FBA not available for requested year") + if method: + message = message.replace("FBA", method) + if year: + message = message.replace("requested year", str(year)) + self.message = message + super().__init__(self.message) + + +class FlowsaMethodNotFoundError(FileNotFoundError): + def __init__(self, method_type=None, method=None): + message = (f"{method_type} method file not found") + if method: + message = " ".join((message, f"for {method}")) + self.message = message + super().__init__(self.message) + + +class APIError(Exception): + def __init__(self, api_source): + message = (f"Key file {api_source} not found. See github wiki for help " + "https://github.com/USEPA/flowsa/wiki/Using-FLOWSA#api-keys") + self.message = message + super().__init__(self.message) + + +class FBSMethodConstructionError(Exception): + """Errors in FBS methods which result in incompatible models""" + def __init__(self, message=None, error_type=None): + if message is None: + message = ("Error in method construction.") + if error_type == 'fxn_call': + message = ("Calling functions in method files must be preceded " + "by '!script_function:'") + self.message = message + super().__init__(self.message) diff --git a/flowsa/fbs_allocation.py b/flowsa/fbs_allocation.py index 6a182d7e7..ae402e879 100644 --- a/flowsa/fbs_allocation.py +++ b/flowsa/fbs_allocation.py @@ -7,6 +7,7 @@ import numpy as np import pandas as pd +import flowsa from flowsa.common import fba_activity_fields, fbs_activity_fields, \ fba_mapped_wsec_default_grouping_fields, fba_wsec_default_grouping_fields, \ check_activities_sector_like, return_bea_codes_used_as_naics @@ -506,8 +507,9 @@ def load_map_clean_fba(method, attr, fba_sourcename, df_year, flowclass, fba = fba.reset_index(drop=True) if len(fba) == 0: - raise Exception('Allocation dataset is length 0; check flow or ' - 'compartment subset for errors') + raise flowsa.exceptions.FBSMethodConstructionError( + message='Allocation dataset is length 0; check flow or ' + 'compartment subset for errors') # load relevant activities if activities are not naics-like try: diff --git a/flowsa/flowbyactivity.py b/flowsa/flowbyactivity.py index aa4e93f41..adce16020 100644 --- a/flowsa/flowbyactivity.py +++ b/flowsa/flowbyactivity.py @@ -11,6 +11,7 @@ import argparse import pandas as pd from urllib import parse +import flowsa from esupy.processed_data_mgmt import write_df_to_file from esupy.remote import make_url_request from flowsa.common import load_api_key, sourceconfigpath, \ @@ -74,11 +75,14 @@ def assemble_urls_for_query(*, source, year, config): userAPIKey = load_api_key(config['api_name']) # (common.py fxn) build_url = build_url.replace("__apiKey__", userAPIKey) - if "url_replace_fxn" in config: - # dynamically import and call on function - urls = config["url_replace_fxn"](build_url=build_url, source=source, - year=year, config=config) + fxn = config.get("url_replace_fxn") + if callable(fxn): + urls = fxn(build_url=build_url, source=source, + year=year, config=config) return urls + elif fxn: + raise flowsa.exceptions.FBSMethodConstructionError( + error_type='fxn_call') else: return [build_url] @@ -108,11 +112,13 @@ def call_urls(*, url_list, source, year, config): resp = make_url_request(url, set_cookies=set_cookies, confirm_gdrive=confirm_gdrive) - if "call_response_fxn" in config: - # dynamically import and call on function - df = config["call_response_fxn"](resp=resp, source=source, - year=year, config=config, - url=url) + fxn = config.get("call_response_fxn") + if callable(fxn): + df = fxn(resp=resp, source=source, year=year, + config=config, url=url) + elif fxn: + raise flowsa.exceptions.FBSMethodConstructionError( + error_type='fxn_call') if isinstance(df, pd.DataFrame): data_frames_list.append(df) elif isinstance(df, list): @@ -131,10 +137,15 @@ def parse_data(*, df_list, source, year, config): :param config: dictionary, FBA yaml :return: df, single df formatted to FBA """ - if "parse_response_fxn" in config: - # dynamically import and call on function - df = config["parse_response_fxn"](df_list=df_list, source=source, - year=year, config=config) + + fxn = config.get("parse_response_fxn") + if callable(fxn): + df = fxn(df_list=df_list, source=source, year=year, config=config) + elif fxn: + raise flowsa.exceptions.FBSMethodConstructionError( + error_type='fxn_call') + # else: + # Handle parse_response_fxn = None return df @@ -187,7 +198,7 @@ def main(**kwargs): # filename if run into error try: config = load_yaml_dict(source, flowbytype='FBA') - except UnboundLocalError: + except FileNotFoundError: log.info(f'Could not find Flow-By-Activity config file for {source}') source = get_flowsa_base_name(sourceconfigpath, source, "yaml") log.info(f'Generating FBA for {source}') diff --git a/flowsa/flowbyfunctions.py b/flowsa/flowbyfunctions.py index 1282c9c8f..d0ecd2ec4 100644 --- a/flowsa/flowbyfunctions.py +++ b/flowsa/flowbyfunctions.py @@ -63,8 +63,9 @@ def filter_by_geoscale(df, geoscale): df = df[df['Location'].isin(fips)].reset_index(drop=True) if len(df) == 0: - log.error("No flows found in the flow dataset at the %s scale", - geoscale) + raise flowsa.exceptions.FBSMethodConstructionError( + message="No flows found in the flow dataset at " + f"the {geoscale} scale") else: return df diff --git a/flowsa/flowbysector.py b/flowsa/flowbysector.py index e967b041e..be6084d0b 100644 --- a/flowsa/flowbysector.py +++ b/flowsa/flowbysector.py @@ -107,12 +107,16 @@ def load_source_dataframe(method, sourcename, source_dict, flows_df = flowsa.getFlowBySector(sourcename) elif source_dict['data_format'] == 'FBS_outside_flowsa': vLog.info("Retrieving flowbysector for datasource %s", sourcename) - flows_df = source_dict["FBS_datapull_fxn"](source_dict, - method, - fbsconfigpath) + fxn = source_dict.get("FBS_datapull_fxn") + if callable(fxn): + flows_df = fxn(source_dict, method, fbsconfigpath) + elif fxn: + raise flowsa.exceptions.FBSMethodConstructionError( + error_type='fxn_call') else: - vLog.error("Data format not specified in method " - "file for datasource %s", sourcename) + raise flowsa.exceptions.FBSMethodConstructionError( + message="Data format not specified in method " + f"file for {sourcename}") return flows_df @@ -132,12 +136,17 @@ def main(**kwargs): fbsconfigpath = kwargs.get('fbsconfigpath') download_FBA_if_missing = kwargs.get('download_FBAs_if_missing') # assign arguments - vLog.info("Initiating flowbysector creation for %s", method_name) + vLog.info(f"Initiating flowbysector creation for {method_name}") # call on method method = load_yaml_dict(method_name, flowbytype='FBS', filepath=fbsconfigpath) # create dictionary of data and allocation datasets - fb = method['source_names'] + try: + fb = method['source_names'] + except KeyError: + log.error("parameter 'source_names' not found in method. " + f"FBS for {method_name} can not be generated.") + return # Create empty list for storing fbs files fbs_list = [] for k, v in fb.items(): @@ -157,18 +166,27 @@ def main(**kwargs): flows = merge_urb_cnty_pct(flows) # clean up fba before mapping, if specified in yaml - if "clean_fba_before_mapping_df_fxn" in v: - vLog.info("Cleaning up %s FlowByActivity", k) - flows = v["clean_fba_before_mapping_df_fxn"](flows) + fxn = v.get("clean_fba_before_mapping_df_fxn") + if callable(fxn): + vLog.info(f"Cleaning up {k} FlowByActivity") + flows = fxn(fba=flows, source_dict=v) + elif fxn: + raise flowsa.exceptions.FBSMethodConstructionError( + error_type='fxn_call') # map flows to federal flow list or material flow list - flows_mapped, mapping_files = \ - map_fbs_flows(flows, k, v, keep_fba_columns=True) + flows_mapped, mapping_files = (map_fbs_flows( + flows, k, v, keep_fba_columns=True, + keep_unmapped_rows=v.get("keep_unmapped_rows", False))) # clean up fba, if specified in yaml - if "clean_fba_df_fxn" in v: - vLog.info("Cleaning up %s FlowByActivity", k) - flows_mapped = v["clean_fba_df_fxn"](flows_mapped) + fxn = v.get("clean_fba_df_fxn") + if callable(fxn): + vLog.info(f"Cleaning up {k} FlowByActivity") + flows_mapped = fxn(fba=flows_mapped, source_dict=v) + elif fxn: + raise flowsa.exceptions.FBSMethodConstructionError( + error_type='fxn_call') # master list of activity names read in from data source ml_act = [] @@ -190,7 +208,7 @@ def main(**kwargs): )].reset_index(drop=True) ml_act.extend(names) - vLog.info("Preparing to handle %s in %s", aset, k) + vLog.info(f"Preparing to handle {aset} in {k}") # subset fba data by activity flows_subset = flows_mapped[ (flows_mapped[fba_activity_fields[0]].isin(names)) | @@ -349,15 +367,19 @@ def main(**kwargs): flows_subset_geo, fbs_sector_subset, aset, k, v, attr, method) - log.info("Completed flowbysector for %s", aset) + log.info(f"Completed flowbysector for {aset}") fbs_list.append(fbs_sector_subset) else: - if 'clean_fbs_df_fxn' in v: - flows = v["clean_fbs_df_fxn"](flows, method) + fxn = v.get("clean_fbs_df_fxn") + if callable(fxn): + flows = fxn(flows, method) + elif fxn: + raise flowsa.exceptions.FBSMethodConstructionError( + error_type='fxn_call') flows = update_geoscale(flows, method['target_geoscale']) # if the loaded flow dt is already in FBS format, # append directly to list of FBS - log.info("Append %s to FBS list", k) + log.info(f"Append {k} to FBS list") # ensure correct field datatypes and add any missing fields flows = clean_df(flows, flow_by_sector_fields, fbs_fill_na_dict) fbs_list.append(flows) @@ -391,7 +413,7 @@ def main(**kwargs): # rename the log file saved to local directory rename_log_file(method_name, meta) log.info('See the Validation log for detailed assessment of ' - 'model results in %s', logoutputpath) + f'model results in {logoutputpath}') if __name__ == '__main__': diff --git a/flowsa/location.py b/flowsa/location.py index 354d6241f..f8bd73f89 100644 --- a/flowsa/location.py +++ b/flowsa/location.py @@ -91,7 +91,7 @@ def apply_county_FIPS(df, year='2015', source_state_abbrev=True): """ # If using 2 letter abbrevations, map to state names if source_state_abbrev: - df['State'] = df['State'].map(abbrev_us_state) + df['State'] = df['State'].map(abbrev_us_state).fillna(df['State']) df['State'] = df.apply(lambda x: clean_str_and_capitalize(x.State), axis=1) df['County'] = df.apply(lambda x: clean_str_and_capitalize(x.County), diff --git a/flowsa/metadata.py b/flowsa/metadata.py index eb1d8734f..7b9b7279d 100644 --- a/flowsa/metadata.py +++ b/flowsa/metadata.py @@ -90,7 +90,8 @@ def return_fbs_method_data(source_name, config): :param config: dictionary, configuration/method file :return: meta object """ - from flowsa.data_source_scripts.stewiFBS import add_stewi_metadata + from flowsa.data_source_scripts.stewiFBS import add_stewi_metadata,\ + add_stewicombo_metadata # load the yaml that lists what additional fbas are # used in creating the fbs @@ -108,8 +109,12 @@ def return_fbs_method_data(source_name, config): for k, v in fb.items(): if k == 'stewiFBS': # get stewi metadata - meta['primary_source_meta'][k] = \ - add_stewi_metadata(v['inventory_dict']) + if v.get('local_inventory_name'): + meta['primary_source_meta'][k] = add_stewicombo_metadata( + v.get('local_inventory_name')) + else: + meta['primary_source_meta'][k] = add_stewi_metadata( + v['inventory_dict']) continue if v['data_format'] in ('FBS', 'FBS_outside_flowsa'): meta['primary_source_meta'][k] = \ diff --git a/flowsa/methods/flowbyactivitymethods/Census_CBP.yaml b/flowsa/methods/flowbyactivitymethods/Census_CBP.yaml index f7a82d684..16629ee9b 100644 --- a/flowsa/methods/flowbyactivitymethods/Census_CBP.yaml +++ b/flowsa/methods/flowbyactivitymethods/Census_CBP.yaml @@ -9,7 +9,7 @@ url: base_url: http://api.census.gov/data/ api_path: __year__/cbp? # __year__ is used to trigger a replace function to substitute in year url_params: - get: __NAICS__,ESTAB #,EMP,PAYANN #replace __NAICS__ based on year of data. >=2017 is NAICS2017, 2012-2016 is NAICS2012 + get: __NAICS__,ESTAB,EMP,PAYANN #replace __NAICS__ based on year of data. >=2017 is NAICS2017, 2012-2016 is NAICS2012 for: county:__countyFIPS__ #retrieves every county in: state:__stateFIPS__ #requests a state 2-digit FIPS code that has to be supplied dynamically key: __apiKey__ # make the __apiKey__ part of the url and substitute in individual api key @@ -26,3 +26,6 @@ years: - 2015 - 2016 - 2017 +- 2018 +- 2019 +- 2020 diff --git a/flowsa/methods/flowbyactivitymethods/EPA_SIT.yaml b/flowsa/methods/flowbyactivitymethods/EPA_SIT.yaml new file mode 100644 index 000000000..36a3f2fee --- /dev/null +++ b/flowsa/methods/flowbyactivitymethods/EPA_SIT.yaml @@ -0,0 +1,231 @@ +author: US Environmental Protection Agency +source_name: State Inventory Tool +source_url: 'https://www.epa.gov/statelocalenergy/state-inventory-and-projection-tool' +bib_id: EPA_SIT +format: xlsm # macro-enabled spreadsheet +url: None +parse_response_fxn: !script_function:EPA_SIT epa_sit_parse +state: 'ME' +file: 'Synthesis Tool.xlsm' +years: +- 2017 +- 2016 +- 2015 +- 2014 +- 2013 +- 2012 +- 2011 +- 2010 +- 2009 +- 2008 +- 2007 +- 2006 +- 2005 +- 2004 +- 2003 +- 2002 +- 2001 +- 2000 +- 1999 +- 1998 +- 1997 +- 1996 +- 1995 +- 1994 +- 1993 +- 1992 +- 1991 +- 1990 + +sheet_dict: + + 'CO2FFC': + nrows: 32 + unit: MMTCO2e + flow: CO2 + headers: + - Residential + - Commercial + - Industrial + - Transportation + - Electric Utilities + - International Bunker Fuels + - TOTAL + + 'IndirectCO2': + nrows: 45 + unit: MMTCO2e + flow: CO2 + headers: + - Residential + - Commercial + - Industrial + - Transportation + - TOTAL + subsubheaders: + - Conventional Boiler Use + - CHP and/or Cogeneration Process + - Process Heating + - Process Cooling and Refrigeration + - Machine Drive + - Electro-Chemical Processes + - Other Process use + - Facility HVAC + - Facility Lighting + - Other Facility Support + - Onsite Transportation + - Other Nonprocess Use + + 'Stationary': + nrows: 15 + unit: MMTCO2e + headers: + - Residential + - Commercial + - Industrial + - Electric Utilities + - TOTAL + skiprowstart: 13 + skiprowend: 15 + subgroup: flow + + 'Mobile Combustion CH4': + sheetname: Mobile Combustion + header: 55 + nrows: 21 + flow: CH4 + unit: MTCO2e + headers: + - Gasoline Highway + - Diesel Highway + - Non-Highway + - Alternative Fuel Vehicles + - Total + + 'Mobile Combustion N2O': + sheetname: Mobile Combustion + header: 82 + nrows: 21 + flow: N2O + unit: MTCO2e + headers: + - Gasoline Highway + - Diesel Highway + - Non-Highway + - Alternative Fuel Vehicles + - Total + + 'Coal': + header: 3 + nrows: 5 + unit: MTCO2e + flow: CH4 + headers: + - Coal Mining + - Abandoned Coal Mines + + 'Gas and Oil': + header: 17 + nrows: 5 + unit: MT + flow: CH4 + headers: + - Natural Gas + - Oil + + 'Natural Gas Flaring': + sheetname: 'Gas and Oil' + header: 24 + nrows: 1 + unit: MMT + flow: CO2 + headers: + - Natural Gas Flaring + + 'IP': + nrows: 20 + unit: MTCO2e + headers: + - Carbon Dioxide Emissions + - Nitrous Oxide Emissions + - HFC, PFC, SF6 and NF3 Emissions + - Total Emissions + subgroup: 'activitybyflow' + + 'Agriculture': + sheetname: Agriculture + header: 26 + nrows: 12 + unit: MMT + headers: + - Carbon Dioxide + - Methane + - Nitrous Oxide + subgroup: 'activitybyflow' + + 'Agricultural Soil Management': + sheetname: Agriculture + tablename: Agricultural Soil Management + header: 41 + nrows: 13 + unit: MT + flow: N2O + headers: + - Direct + - Indirect + - TOTAL + subsubheaders: + - Fertilizer Runoff/Leached + - Manure Runoff/Leached + + 'Land-Use Change and Forest Emissions and Sequestration': + sheetname: Forest Management + header: 8 + nrows: 19 + unit: MMTCO2E + flow: CO2E + headers: + - Forest Carbon Flux + - Urban Trees + - Landfilled Yard Trimmings and Food Scraps + - Forest Fires + - N2O from Settlement Soils + - Agricultural Soil Carbon Flux + + 'Emissions from Landfills': + sheetname: Waste + tablename: Emissions from Landfills + header: 13 + nrows: 9 + unit: MTCO2E + flow: CH4 + headers: + - Potential CH4 + - CH4 Avoided + - Oxidation at MSW Landfills + - Oxidation at Industrial Landfills + - Total CH4 Emissions + + 'Waste Combustion': + sheetname: Waste + tablename: Emissions from Waste Combustion + header: 25 + nrows: 7 + unit: MTCO2e + headers: + - CO2 + - N2O + - CH4 + - Total CO2, N2O, CH4 Emissions + subgroup: 'activitybyflow' + + 'Wastewater': + header: 3 + nrows: 8 + unit: MMTCO2e + headers: + - Municipal CH4 + - Municipal N2O + - Industrial CH4 + - Total Emissions + subgroup: 'activitybyflow' \ No newline at end of file diff --git a/flowsa/methods/flowbyactivitymethods/EPA_StateGHGI.yaml b/flowsa/methods/flowbyactivitymethods/EPA_StateGHGI.yaml new file mode 100644 index 000000000..026c671b4 --- /dev/null +++ b/flowsa/methods/flowbyactivitymethods/EPA_StateGHGI.yaml @@ -0,0 +1,39 @@ +author: US Environmental Protection Agency +source_name: 'State Greenhouse Gas Inventories' +source_url: 'https://www.epa.gov/ghgemissions/state-ghg-emissions-and-removals' +bib_id: '' +format: json +url: None +parse_response_fxn: !script_function:EPA_StateGHGI epa_state_ghgi_parse +file: "W_INV_FACTS_ACTIVE_GHG_CO2E.json" +years: +- 2019 +- 2018 +- 2017 +- 2016 +- 2015 +- 2014 +- 2013 +- 2012 +- 2011 +- 2010 +- 2009 +- 2008 +- 2007 +- 2006 +- 2005 +- 2004 +- 2003 +- 2002 +- 2001 +- 2000 +- 1999 +- 1998 +- 1997 +- 1996 +- 1995 +- 1994 +- 1993 +- 1992 +- 1991 +- 1990 diff --git a/flowsa/methods/flowbyactivitymethods/stateio_Common.yaml b/flowsa/methods/flowbyactivitymethods/stateio_Common.yaml new file mode 100644 index 000000000..9a61c425e --- /dev/null +++ b/flowsa/methods/flowbyactivitymethods/stateio_Common.yaml @@ -0,0 +1,18 @@ +author: Li et al. +# source_name: stateio_Make_Summary # Update for each dataset +source_url: https://github.com/USEPA/stateior +original_data_download_date: +# bib_id: # Update for each dataset +api_name: None +api_key_required: False +format: rds # R data format +url: None # Uses esupy data retrieval +parse_response_fxn: !script_function:stateio parse_statior +years: +- 2012 +- 2013 +- 2014 +- 2015 +- 2016 +- 2017 +# datatype: 'State_Summary_Make' # Update for each dataset diff --git a/flowsa/methods/flowbyactivitymethods/stateio_Industry_GO.yaml b/flowsa/methods/flowbyactivitymethods/stateio_Industry_GO.yaml new file mode 100644 index 000000000..6445af195 --- /dev/null +++ b/flowsa/methods/flowbyactivitymethods/stateio_Industry_GO.yaml @@ -0,0 +1,4 @@ +!include:stateio_Common.yaml +source_name: stateio_Industry_GO +bib_id: +datatype: 'State_Summary_IndustryOutput' diff --git a/flowsa/methods/flowbyactivitymethods/stateio_Make_Summary.yaml b/flowsa/methods/flowbyactivitymethods/stateio_Make_Summary.yaml new file mode 100644 index 000000000..b8b8c62fb --- /dev/null +++ b/flowsa/methods/flowbyactivitymethods/stateio_Make_Summary.yaml @@ -0,0 +1,4 @@ +!include:stateio_Common.yaml +source_name: stateio_Make_Summary +bib_id: +datatype: 'State_Summary_Make' diff --git a/flowsa/methods/flowbyactivitymethods/stateio_Use_Summary.yaml b/flowsa/methods/flowbyactivitymethods/stateio_Use_Summary.yaml new file mode 100644 index 000000000..da1f61c89 --- /dev/null +++ b/flowsa/methods/flowbyactivitymethods/stateio_Use_Summary.yaml @@ -0,0 +1,4 @@ +!include:stateio_Common.yaml +source_name: stateio_Use_Summary +bib_id: +datatype: 'State_Summary_Use' diff --git a/flowsa/methods/flowbysectormethods/CAP_HAP_national_2017.yaml b/flowsa/methods/flowbysectormethods/CAP_HAP_national_2017.yaml index 0cebf55bd..2f9153568 100644 --- a/flowsa/methods/flowbysectormethods/CAP_HAP_national_2017.yaml +++ b/flowsa/methods/flowbysectormethods/CAP_HAP_national_2017.yaml @@ -198,9 +198,8 @@ source_names: FBS_datapull_fxn: !script_function:stewiFBS stewicombo_to_sector inventory_dict: {"NEI":"2017", "TRI":"2017"} local_inventory_name: 'NEI_TRI_air_2017' + clean_fbs_df_fxn: !script_function:EPA_NEI drop_GHGs compartments: - 'air' - functions: - - 'drop_GHGs' reassign_process_to_sectors: - 'airplane_emissions' diff --git a/flowsa/methods/flowbysectormethods/CNHW_state_2014.yaml b/flowsa/methods/flowbysectormethods/CNHW_state_2014.yaml index fcb9a45e1..ebd231ca8 100644 --- a/flowsa/methods/flowbysectormethods/CNHW_state_2014.yaml +++ b/flowsa/methods/flowbysectormethods/CNHW_state_2014.yaml @@ -1,4 +1,4 @@ -!include:BEA_summary_target.yaml +!include:USEEIO_summary_target.yaml # target_sector_level: NAICS_6 # target_sector_source: NAICS_2012_Code target_geoscale: state diff --git a/flowsa/methods/flowbysectormethods/README.md b/flowsa/methods/flowbysectormethods/README.md index 9196273a7..9c348f542 100644 --- a/flowsa/methods/flowbysectormethods/README.md +++ b/flowsa/methods/flowbysectormethods/README.md @@ -1,108 +1,110 @@ # FlowBySector method yaml term descriptions -Description of parameters in flowbysectormethods yamls. All values are +Description of parameters in flowbysectormethods yamls. All values are strings unless noted. ## Terms ### Target FBS output specifications -- _target_sector_level_: specify desired sector aggregation (NAICS_2, +- _target_sector_level_: specify desired sector aggregation (NAICS_2, NAICS_3, NAICS_4, NAICS_5, NAICS_6) -- _target_sector_source_: specify NAICS version 2007, 2012, 2017 (ex. - NAICS_2012_Code). At this time, only NAICS_2012_Code is supported. -- _target_geoscale_: level of geographic aggregation in output parquet +- _target_sector_source_: specify NAICS version 2007, 2012, 2017 (ex. + NAICS_2012_Code). At this time, only NAICS_2012_Code is supported. +- _target_geoscale_: level of geographic aggregation in output parquet (national, state, or county) -- _download_if_missing_: (optional) Add and set to 'True' if you would like - to download all required FBAs from Data Commons rather than generating +- _download_if_missing_: (optional) Add and set to 'True' if you would like + to download all required FBAs from Data Commons rather than generating FBAs locally. ### Source specifications (in FBA format) -- _source_names_: The name of the FBS dataset or the FBA dataset requiring +- _source_names_: The name of the FBS dataset or the FBA dataset requiring allocation to sectors -- _data_format_: 'FBA', 'FBS', 'FBS_outside_flowsa', loads a FlowByActivity - or a FlowBySector parquet stored in flowsa, or calls on a specified +- _data_format_: 'FBA', 'FBS', 'FBS_outside_flowsa', loads a FlowByActivity + or a FlowBySector parquet stored in flowsa, or calls on a specified function to load data from outside flowsa in FBS format -- _class_: a text string in 'Class' column of flowbyactivity (ex. Water), - see class types in +- _class_: a text string in 'Class' column of flowbyactivity (ex. Water), + see class types in [source_catalog.yaml](https://github.com/USEPA/flowsa/blob/master/flowsa/data/source_catalog.yaml) - _geoscale_to_use_: the geoscale of the FBA set to use for sector allocation (national, state, or county) - _year_: year of available dataset (ex. 2015) -- _activity_to_sector_mapping_: (optional) name of activity to sector +- _activity_to_sector_mapping_: (optional) name of activity to sector mapping file, if not provided will use the source name -- _apply_urban_rural_: Assign flow quantities as urban or rural based on +- _apply_urban_rural_: Assign flow quantities as urban or rural based on population density by FIPS. -- _clean_fba_before_mapping_df_fxn_: (optional) calls on function in the - source.py file to clean up/modify the FBA data prior to mapping flows. +- _clean_fba_before_mapping_df_fxn_: (optional) calls on function in the + source.py file to clean up/modify the FBA data prior to mapping flows. Function is called using the `!script_function:` tag. -- _clean_fba_df_fxn_: (optional) calls on function in the source.py file to - clean up/modify the FBA data prior to allocating data to sectors. +- _clean_fba_df_fxn_: (optional) calls on function in the source.py file to + clean up/modify the FBA data prior to allocating data to sectors. Function is called using the `!script_function:` tag. -- _clean_fba_w_sec_df_fxn_: (optional) calls on function in the source.py - file to clean up/modify the FBA dataframe, after sector columns are added - but prior to allocating data to sectors. Function is called using +- _clean_fba_w_sec_df_fxn_: (optional) calls on function in the source.py + file to clean up/modify the FBA dataframe, after sector columns are added + but prior to allocating data to sectors. Function is called using the`!script_function:` tag. -- _fedefl_mapping_: (optional) name of mapping file in FEDEFL. If not +- _fedefl_mapping_: (optional) name of mapping file in FEDEFL. If not supplied will use the source_names -- _mfl_mapping_: (optional, should not be used if fedefl_mapping is used) +- _mfl_mapping_: (optional, should not be used if fedefl_mapping is used) name of mapping file for Material Flow List. +- _keep_unmapped_rows_: (optional) default is False, if True will maintain any + flows not found in mapping files. ### Activity set specifications -- _activity_sets_: A subset of the FBA dataset and the method and +- _activity_sets_: A subset of the FBA dataset and the method and allocation datasets used to create an FBS - _names_: (list) specify the subset of the FBA to allocate based on values in the Activity Produced/Consumed By fields. To use an external activity set . - csv file, use the tag `!from_index:file_name.csv`, then give the name (e.g., + csv file, use the tag `!from_index:file_name.csv`, then give the name (e.g., `activity_set_1`) of the activity set as found in the csv file. -- _source_flows_: (list, optional) specify the 'FlowName'(s) from the FBA +- _source_flows_: (list, optional) specify the 'FlowName'(s) from the FBA to use. If not provided, all flows are used. -- _allocation_method_: currently written for 'direct', - 'allocation_function', 'proportional', and 'proportional-flagged'. See +- _allocation_method_: currently written for 'direct', + 'allocation_function', 'proportional', and 'proportional-flagged'. See descriptions below. - _allocation_source_: The primary data source used to allocate main FBA for specified activity to sectors -- _literature_sources_: (optional) Specific functions that contain values - from literature used to modify FBA data. -- _activity_to_sector_mapping_: (optional) name of activity to sector +- _literature_sources_: (optional) Specific functions that contain values + from literature used to modify FBA data. +- _activity_to_sector_mapping_: (optional) name of activity to sector mapping file, if not provided will use the source name -- _allocation_source_class_: specific 'FlowClass' found in the allocation +- _allocation_source_class_: specific 'FlowClass' found in the allocation source flowbyactivity parquet -- _allocation_source_year_: specific to the allocation datasets, use the +- _allocation_source_year_: specific to the allocation datasets, use the year relevant to the main FBA dataframe -- _allocation_flow_: (list) the relevant 'FlowName' values, as found in the +- _allocation_flow_: (list) the relevant 'FlowName' values, as found in the source flowbyactivity parquet. Use 'None' to capture all flows. -- _allocation_compartment_: (list) the relevant 'Compartment' values, as - found in the source flowbyactivity parquet. Use 'None' to capture all +- _allocation_compartment_: (list) the relevant 'Compartment' values, as + found in the source flowbyactivity parquet. Use 'None' to capture all compartments. -- _allocation_from_scale_: national, state, or county - dependent on +- _allocation_from_scale_: national, state, or county - dependent on allocation source, as not every level exits for sources -- _allocation_fba_load_scale_: (optional) Can indicate geographic level of +- _allocation_fba_load_scale_: (optional) Can indicate geographic level of FBA to load, helpful when an FBA ia large -- _clean_allocation_fba_: (optional) Function to clean up the allocation - FBA, as defined in the source.py file. Function is called using +- _clean_allocation_fba_: (optional) Function to clean up the allocation + FBA, as defined in the source.py file. Function is called using the`!script_function:` tag. -- _clean_allocation_fba_w_sec_: (optional) Function to clean up the - allocation FBA, after allocation activities are assigned SectorProducedBy - and SectorConsumedBy columns. Function is called using +- _clean_allocation_fba_w_sec_: (optional) Function to clean up the + allocation FBA, after allocation activities are assigned SectorProducedBy + and SectorConsumedBy columns. Function is called using the`!script_function:` tag. -- _allocation_map_to_flow_list_: (optional) If the allocation df and source +- _allocation_map_to_flow_list_: (optional) If the allocation df and source df need to be matched on Context and/or Flowable, set to 'True' - _helper_source_: (optional) secondary df for sector allocation -- _helper_method_: currently written for 'multiplication', 'proportional', +- _helper_method_: currently written for 'multiplication', 'proportional', and 'proportional-flagged' -- _helper_activity_to_sector_mapping_: (optional) name of activity to +- _helper_activity_to_sector_mapping_: (optional) name of activity to sector mapping file, if not provided will use the source name -- _helper_source_class_: specific 'FlowClass' found in the allocation +- _helper_source_class_: specific 'FlowClass' found in the allocation source flowbyactivity parquet -- _helper_source_year_: specific to the allocation datasets, use the year +- _helper_source_year_: specific to the allocation datasets, use the year relevant to the main FBA dataframe -- _helper_flow_: (list) the relevant 'FlowName' values, as found in the +- _helper_flow_: (list) the relevant 'FlowName' values, as found in the source flowbyactivity parquet -- _helper_from_scale_: national, state, or county - dependent on allocation +- _helper_from_scale_: national, state, or county - dependent on allocation source, as not every level exits for sources -- _clean_helper_fba_: (optional) Function to clean up the helper FBA. +- _clean_helper_fba_: (optional) Function to clean up the helper FBA. Function is called using the`!script_function:` tag. -- _clean_helper_fba_wsec_: (optional) Function to clean up the helper FBA, - after allocation activities are assigned SectorProducedBy and - SectorConsumedBy columns. Function is called using +- _clean_helper_fba_wsec_: (optional) Function to clean up the helper FBA, + after allocation activities are assigned SectorProducedBy and + SectorConsumedBy columns. Function is called using the`!script_function:` tag. ### Source specifications (in FBS format) @@ -110,25 +112,25 @@ If source data format is specified as 'FBS': - _source_names_: The name of the FBS dataset - _data_format_: 'FBS', loads a FlowBySector - _year_: year of available dataset (ex. 2015) -- _clean_fbs_df_fxn_: (optional) apply function to clean the FBS after it +- _clean_fbs_df_fxn_: (optional) apply function to clean the FBS after it is accessed. Function is called using the`!script_function:` tag. ### FBS_outside_flows specifications If source data_format is specified as `FBS_outside_flowsa`: -- _FBS_datapull_fxn_: name of the function to generate the FBS. Function is +- _FBS_datapull_fxn_: name of the function to generate the FBS. Function is called using the`!script_function:` tag. - _parameters_: (list) parameters to pass into the function ## Method Descriptions -- allocation_function: Activities are assigned to sectors using a specified +- allocation_function: Activities are assigned to sectors using a specified function -- direct: Activities are directly assigned to sectors using the source to +- direct: Activities are directly assigned to sectors using the source to NAICS crosswalk -- multiplication: Multiply the values in the allocation data source with +- multiplication: Multiply the values in the allocation data source with values sharing the same sectors in the helper allocation data source -- proportional: Activities are proportionally allocated to sectors using +- proportional: Activities are proportionally allocated to sectors using specified allocation data source -- proportional-flagged: Activities that are flagged (assigned a value of - '1') are proportionally allocated to sectors using a specified allocation - data source. Activities that are not flagged (assigned a value of '0') +- proportional-flagged: Activities that are flagged (assigned a value of + '1') are proportionally allocated to sectors using a specified allocation + data source. Activities that are not flagged (assigned a value of '0') are directly assigned to sectors. diff --git a/flowsa/methods/flowbysectormethods/USEEIO_summary_target.yaml b/flowsa/methods/flowbysectormethods/USEEIO_summary_target.yaml new file mode 100644 index 000000000..b3a51eef3 --- /dev/null +++ b/flowsa/methods/flowbysectormethods/USEEIO_summary_target.yaml @@ -0,0 +1,13 @@ +# This file can be used to set up target NAICS for standard sectors +# in a USEEIO summary level model. NAICS are targeted to enable 1:1 +# correspondence between NAICS and BEA Summary sectors + +# To use in a FBS method add +# !include:USEEIO_summary_target.yaml +# to the top of the method replacing the three parameters below + +target_sector_level: NAICS_3 +target_subset_sector_level: {NAICS_4: ['221', '336', '541']} +# In USEEIO models 221 (Utilities) is disaggregated to 2211, 2212, and 2213 +# '336' and '541' carry over from the BEA summary sectors +target_sector_source: NAICS_2012_Code diff --git a/flowsa/methods/flowbysectormethods/Water_state_2015_m1.yaml b/flowsa/methods/flowbysectormethods/Water_state_2015_m1.yaml index 6bf32ffdc..d78578630 100644 --- a/flowsa/methods/flowbysectormethods/Water_state_2015_m1.yaml +++ b/flowsa/methods/flowbysectormethods/Water_state_2015_m1.yaml @@ -1,4 +1,4 @@ -!include:BEA_summary_target.yaml +!include:USEEIO_summary_target.yaml target_geoscale: state source_names: "USGS_NWIS_WU": diff --git a/flowsa/schema.py b/flowsa/schema.py index 2961aadba..85873a83c 100644 --- a/flowsa/schema.py +++ b/flowsa/schema.py @@ -170,3 +170,7 @@ 'ConsumedBy': [{'flowbyactivity': 'ActivityConsumedBy'}, {'flowbysector': 'SectorConsumedBy'}] } + +dq_fields = ['MeasureofSpread', 'Spread', 'DistributionType', 'Min', + 'Max', 'DataReliability', 'DataCollection', 'TemporalCorrelation', + 'GeographicalCorrelation', 'TechnologicalCorrelation'] diff --git a/flowsa/sectormapping.py b/flowsa/sectormapping.py index d869897f4..c4e1353c1 100644 --- a/flowsa/sectormapping.py +++ b/flowsa/sectormapping.py @@ -8,10 +8,11 @@ import pandas as pd import numpy as np from esupy.mapping import apply_flow_mapping +import flowsa from flowsa.common import get_flowsa_base_name, \ return_true_source_catalog_name, check_activities_sector_like, \ load_yaml_dict, fba_activity_fields, SECTOR_SOURCE_NAME -from flowsa.schema import activity_fields +from flowsa.schema import activity_fields, dq_fields from flowsa.settings import log from flowsa.flowbyfunctions import fbs_activity_fields, load_crosswalk from flowsa.validation import replace_naics_w_naics_from_another_year @@ -439,3 +440,98 @@ def get_sector_list(sector_level, secondary_sector_level_dict=None): sector_list = sector_list + sector_add return sector_list + + +def map_to_BEA_sectors(fbs_load, region, io_level, year): + """ + Map FBS sectors from NAICS to BEA, allocating by gross industry output. + + :param fbs_load: df completed FlowBySector collapsed to single 'Sector' + :param region: str, 'state' or 'national' + :param io_level: str, 'summary' or 'detail' + :param year: year for industry output + """ + from flowsa.sectormapping import get_activitytosector_mapping + + bea = get_BEA_industry_output(region, io_level, year) + + if io_level == 'summary': + mapping_col = 'BEA_2012_Summary_Code' + elif io_level == 'detail': + mapping_col = 'BEA_2012_Detail_Code' + + # Prepare NAICS:BEA mapping file + mapping = (load_crosswalk('BEA') + .rename(columns={mapping_col: 'BEA', + 'NAICS_2012_Code': 'Sector'})) + mapping = (mapping.drop( + columns=mapping.columns.difference(['Sector','BEA'])) + .drop_duplicates(ignore_index=True) + .dropna(subset=['Sector'])) + mapping['Sector'] = mapping['Sector'].astype(str) + + # Create allocation ratios where one to many NAICS:BEA + dup = mapping[mapping['Sector'].duplicated(keep=False)] + dup = dup.merge(bea, how='left', on='BEA') + dup['Allocation'] = dup['Output']/dup.groupby( + ['Sector','Location']).Output.transform('sum') + + # Update and allocate to sectors + fbs = (fbs_load.merge( + mapping.drop_duplicates(subset='Sector', keep=False), + how='left', + on='Sector')) + fbs = fbs.merge(dup.drop(columns='Output'), + how='left', on=['Sector', 'Location'], + suffixes=(None, '_y')) + fbs['Allocation'] = fbs['Allocation'].fillna(1) + fbs['BEA'] = fbs['BEA'].fillna(fbs['BEA_y']) + fbs['FlowAmount'] = fbs['FlowAmount'] * fbs['Allocation'] + + fbs = (fbs.drop(columns=dq_fields + + ['Sector', 'SectorSourceName', + 'BEA_y', 'Allocation'], errors='ignore') + .rename(columns={'BEA':'Sector'})) + + if (abs(1-(sum(fbs['FlowAmount']) / + sum(fbs_load['FlowAmount'])))) > 0.005: + log.warning('Data loss upon BEA mapping') + + return fbs + + +def get_BEA_industry_output(region, io_level, year): + """ + Get FlowByActivity for industry output from state or national datasets + :param region: str, 'state' or 'national' + :param io_level: str, 'summary' or 'detail' + :param year: year for industry output + """ + if region == 'state': + fba = 'stateio_Industry_GO' + if io_level == 'detail': + raise TypeError ('detail models not available for states') + elif region == 'national': + fba = 'BEA_GDP_GrossOutput' + + # Get output by BEA sector + bea = flowsa.getFlowByActivity(fba, year) + bea = ( + bea.drop(columns=bea.columns.difference( + ['FlowAmount','ActivityProducedBy','Location'])) + .rename(columns={'FlowAmount':'Output', + 'ActivityProducedBy': 'BEA'})) + + # If needed, aggregate from detial to summary + if region == 'national' and io_level == 'summary': + bea_mapping = (load_crosswalk('BEA') + [['BEA_2012_Detail_Code','BEA_2012_Summary_Code']] + .drop_duplicates() + .rename(columns={'BEA_2012_Detail_Code': 'BEA'})) + bea = (bea.merge(bea_mapping, how='left', on='BEA') + .drop(columns=['BEA']) + .rename(columns={'BEA_2012_Summary_Code': 'BEA'})) + bea = (bea.groupby(['BEA','Location']).agg({'Output': 'sum'}) + .reset_index()) + + return bea diff --git a/flowsa/test_examples.py b/flowsa/test_examples.py index 2f1a71210..97410a4a5 100644 --- a/flowsa/test_examples.py +++ b/flowsa/test_examples.py @@ -24,6 +24,7 @@ def test_generate_fbs(): """Generate all FBS from methods in repo.""" for m in flowsa.seeAvailableFlowByModels("FBS", print_method=False): if m not in ['BEA_summary_target', + 'USEEIO_summary_target', 'Electricity_gen_emissions_national_2016']: print("--------------------------------\n" f"Method: {m}\n" diff --git a/flowsa/test_methods.py b/flowsa/test_methods.py index f8fd84b4d..092a1b48c 100644 --- a/flowsa/test_methods.py +++ b/flowsa/test_methods.py @@ -4,6 +4,7 @@ """ import pytest from flowsa import seeAvailableFlowByModels +import flowsa.exceptions from flowsa.flowbyactivity import load_yaml_dict, assemble_urls_for_query,\ call_urls @@ -18,7 +19,6 @@ def test_FBA_urls(): year = max(config['years']) if ((config.get('url', 'None') == 'None') or - (config.get('api_key_required', False)) or (m == 'EPA_EQUATES')): continue @@ -33,6 +33,9 @@ def test_FBA_urls(): config=config) call_urls(url_list=urls, source=m, year=str(year), config=config) + except flowsa.exceptions.APIError: + print('API Key required, skipping url') + continue except Exception: error_list.append(m) if error_list: diff --git a/flowsa/validation.py b/flowsa/validation.py index 77871ee4c..01a2446e7 100644 --- a/flowsa/validation.py +++ b/flowsa/validation.py @@ -7,8 +7,10 @@ import pandas as pd import numpy as np +import flowsa from flowsa.flowbyfunctions import aggregator, create_geoscale_list,\ - subset_df_by_geoscale, sector_aggregation, subset_df_by_sector_lengths + subset_df_by_geoscale, sector_aggregation, collapse_fbs_sectors,\ + subset_df_by_sector_lengths from flowsa.dataclean import replace_strings_with_NoneType, \ replace_NoneType_with_empty_cells from flowsa.common import sector_level_key, \ @@ -1069,3 +1071,88 @@ def compare_df_units(df1_load, df2_load): # if list is not empty, print warning that units are different if list_comp: log.info('Merging df with %s and df with %s units', df1, df2) + + +def calculate_industry_coefficients(fbs_load, year,region, + io_level, impacts=False): + """ + Generates sector coefficients (flow/$) for all sectors for all locations. + + :param fbs_load: flow by sector method + :param year: year for industry output dataset + :param region: str, 'state' or 'national' + :param io_level: str, 'summary' or 'detail' + :param impacts: bool, True to apply and aggregate on impacts + False to compare flow/contexts + """ + from flowsa.sectormapping import map_to_BEA_sectors,\ + get_BEA_industry_output + + fbs = collapse_fbs_sectors(fbs_load) + + fbs = map_to_BEA_sectors(fbs, region, io_level, year) + + inventory = not(impacts) + if impacts: + try: + import lciafmt + fbs_summary = (lciafmt.apply_lcia_method(fbs, 'TRACI2.1') + .rename(columns={'FlowAmount': 'InvAmount', + 'Impact': 'FlowAmount'})) + groupby_cols = ['Location', 'Sector', + 'Indicator', 'Indicator unit'] + sort_by_cols = ['Indicator', 'Sector', 'Location'] + except ImportError: + log.warning('lciafmt not installed') + inventory = True + except AttributeError: + log.warning('check lciafmt branch') + inventory = True + + if inventory: + fbs_summary = fbs.copy() + groupby_cols = ['Location', 'Sector', + 'Flowable', 'Context', 'Unit'] + sort_by_cols = ['Context', 'Flowable', + 'Sector', 'Location'] + + # Update location if needed prior to aggregation + if region == 'national': + fbs_summary["Location"] = US_FIPS + + fbs_summary = (fbs_summary.groupby(groupby_cols) + .agg({'FlowAmount': 'sum'}). + reset_index()) + + bea = get_BEA_industry_output(region, io_level, year) + + # Add sector output and assign coefficients + fbs_summary = fbs_summary.merge(bea.rename( + columns={'BEA': 'Sector'}), how = 'left', + on=['Sector','Location']) + fbs_summary['Coefficient'] = (fbs_summary['FlowAmount'] / + fbs_summary['Output']) + fbs_summary = fbs_summary.sort_values(by=sort_by_cols) + + return fbs_summary + + +if __name__ == "__main__": + df1 = calculate_industry_coefficients( + flowsa.getFlowBySector('Water_national_2015_m1'), 2015, + "national", "summary", False) + df2 = calculate_industry_coefficients( + flowsa.getFlowBySector('GRDREL_national_2017'), 2017, + "national", "summary", True) + df3 = calculate_industry_coefficients( + flowsa.getFlowBySector('GRDREL_national_2017'), 2017, + "national", "detail", True) + df4 = calculate_industry_coefficients( + flowsa.getFlowBySector('GRDREL_state_2017'), 2017, + "national", "detail", True) + try: + df5 = calculate_industry_coefficients( + flowsa.getFlowBySector('GRDREL_state_2017'), 2017, + "state", "detail", True) + except TypeError: + df5 = None diff --git a/scripts/FlowByActivity_Crosswalks/write_Crosswalk_BEA_2012_Detail.py b/scripts/FlowByActivity_Crosswalks/write_Crosswalk_BEA_2012.py similarity index 66% rename from scripts/FlowByActivity_Crosswalks/write_Crosswalk_BEA_2012_Detail.py rename to scripts/FlowByActivity_Crosswalks/write_Crosswalk_BEA_2012.py index 0b81ce25a..292486b32 100644 --- a/scripts/FlowByActivity_Crosswalks/write_Crosswalk_BEA_2012_Detail.py +++ b/scripts/FlowByActivity_Crosswalks/write_Crosswalk_BEA_2012.py @@ -1,26 +1,26 @@ -# write_Crosswalk_BEA_2012_Detail.py (scripts) +# write_Crosswalk_BEA_2012.py (scripts) # !/usr/bin/env python3 # coding=utf-8 """ -Create a crosswalk linking BEA to NAICS for 2012 Detail +Create a crosswalk linking BEA to NAICS for 2012 for any level """ from flowsa.common import load_crosswalk from flowsa.settings import datapath -if __name__ == '__main__': +def write_BEA_crosswalk(level='Detail'): cw_load = load_crosswalk('BEA') - cw = cw_load[['BEA_2012_Detail_Code', + cw = cw_load[[f'BEA_2012_{level}_Code', 'NAICS_2012_Code']].drop_duplicates().reset_index(drop=True) # drop all rows with naics >6 cw = cw[cw['NAICS_2012_Code'].apply(lambda x: len(str(x)) == 6)].reset_index(drop=True) df = cw.rename(columns={"NAICS_2012_Code": "Sector", - "BEA_2012_Detail_Code":"Activity"}) + f"BEA_2012_{level}_Code":"Activity"}) df['SectorSourceName'] = 'NAICS_2012_Code' - df['ActivitySourceName'] = 'BEA_2012_Detail_Code' + df['ActivitySourceName'] = f'BEA_2012_{level}_Code' df.dropna(subset=["Sector"], inplace=True) # assign sector type df['SectorType'] = None @@ -32,4 +32,8 @@ df = df[['ActivitySourceName', 'Activity', 'SectorSourceName', 'Sector', 'SectorType']] # save as csv df.to_csv(datapath + "activitytosectormapping/" + - "NAICS_Crosswalk_BEA_2012_Detail.csv", index=False) + f"NAICS_Crosswalk_BEA_2012_{level}.csv", index=False) + +if __name__ == '__main__': + write_BEA_crosswalk('Detail') + write_BEA_crosswalk('Summary') \ No newline at end of file diff --git a/setup.py b/setup.py index 39bc95830..f910d3ab0 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name='flowsa', - version='1.2.1', + version='1.2.2', packages=find_packages(), package_dir={'flowsa': 'flowsa'}, include_package_data=True,