Determined that Frenck's Spook Add-On is best. Used python to format data payload for recorder statistics import service.

data_clean.py

@@ -0,0 +1,43 @@
+import pandas as pd
+
+file_path = './input/raw/postgres/total_electric_consumption_pg_id_40.csv'
+df = pd.read_csv(file_path)
+
+# Select only the necessary columns
+df = df[['start_ts', 'sum', 'state']]
+
+# Convert 'start_ts' column to datetime
+df['start_ts'] = pd.to_datetime(df['start_ts'], unit='s')
+
+# Save the data in the desired format to a text file
+output_file_path = './output/total_electric_consumption_pg_id_40.txt'
+
+with open(output_file_path, 'w') as f:
+    for index, row in df.iterrows():
+        f.write(f"    - start: \"{row['start_ts']}+00:00\"\n")
+        f.write(f"      state: {row['state']}\n")
+        f.write(f"      sum: {row['sum']}\n")
+
+print(f"Data saved to: {output_file_path}")
+
+
+
+file_path = './input/raw/postgres/total_electric_consumption_cost_pg_id_41.csv'
+df = pd.read_csv(file_path)
+
+# Select only the necessary columns
+df = df[['start_ts', 'sum', 'state']]
+
+# Convert 'start_ts' column to datetime
+df['start_ts'] = pd.to_datetime(df['start_ts'], unit='s')
+
+# Save the data in the desired format to a text file
+output_file_path = './output/total_electric_consumption_cost_pg_id_41.txt'
+
+with open(output_file_path, 'w') as f:
+    for index, row in df.iterrows():
+        f.write(f"    - start: \"{row['start_ts']}+00:00\"\n")
+        f.write(f"      state: {row['state']}\n")
+        f.write(f"      sum: {row['sum']}\n")
+
+print(f"Data saved to: {output_file_path}")

input/raw/sqlite/statistics_meta-export.csv

@@ -1,68 +0,0 @@
-id,statistic_id,source,unit_of_measurement,has_mean,has_sum,name
-1,sensor.freebsd_router_kib_s_received,recorder,KiB/s,True,False,
-2,sensor.freebsd_router_kib_s_sent,recorder,KiB/s,True,False,
-3,sensor.disk_use_percent_config,recorder,%,True,False,
-4,sensor.disk_use,recorder,GiB,True,False,
-5,sensor.disk_free,recorder,GiB,True,False,
-6,sensor.memory_use_percent,recorder,%,True,False,
-7,sensor.memory_use,recorder,MiB,True,False,
-8,sensor.memory_free,recorder,MiB,True,False,
-9,sensor.swap_use_percent,recorder,%,True,False,
-10,sensor.swap_use,recorder,MiB,True,False,
-11,sensor.swap_free,recorder,MiB,True,False,
-12,sensor.load_1m,recorder,,True,False,
-13,sensor.load_5m,recorder,,True,False,
-14,sensor.load_15m,recorder,,True,False,
-15,sensor.network_in_enp0s18,recorder,MiB,False,True,
-16,sensor.network_out_enp0s18,recorder,MiB,False,True,
-17,sensor.network_throughput_in_enp0s18,recorder,MB/s,True,False,
-18,sensor.network_throughput_out_enp0s18,recorder,MB/s,True,False,
-19,sensor.packets_in_enp0s18,recorder,,False,True,
-20,sensor.packets_out_enp0s18,recorder,,False,True,
-21,sensor.processor_use,recorder,%,True,False,
-22,sensor.hvac_cooling_time_yesterday,recorder,h,True,False,
-23,sensor.sump_pump_run_counter_today,recorder,,True,False,
-24,sensor.hvac_cooling_time_today,recorder,h,True,False,
-25,sensor.hvac_heating_time_yesterday,recorder,h,True,False,
-26,sensor.sump_pump_run_counter_yesterday,recorder,,True,False,
-27,sensor.hvac_heating_time_today,recorder,h,True,False,
-28,sensor.total_panel_power,recorder,W,True,False,
-29,sensor.total_panel_energy,recorder,kWh,True,False,
-30,sensor.total_electric_consumption_cost,recorder,USD,False,True,
-31,sensor.total_electric_consumption,recorder,kWh,False,True,
-32,sensor.service_leg_02_power,recorder,W,True,False,
-33,sensor.service_leg_02_energy,recorder,kWh,False,True,
-34,sensor.service_leg_02_energy_returned,recorder,kWh,False,True,
-35,sensor.service_leg_02_voltage,recorder,V,True,False,
-36,sensor.service_leg_01_power,recorder,W,True,False,
-37,sensor.service_leg_01_energy,recorder,kWh,False,True,
-38,sensor.service_leg_01_energy_returned,recorder,kWh,False,True,
-39,sensor.service_leg_01_voltage,recorder,V,True,False,
-40,sensor.qq8j_battery,recorder,%,True,False,
-41,sensor.qq8j_temperature,recorder,°F,True,False,
-42,sensor.my_ecobee_current_temperature,recorder,°F,True,False,
-43,sensor.my_ecobee_current_humidity,recorder,%,True,False,
-44,sensor.tasmota_signal,recorder,dBm,True,False,
-45,sensor.tasmota_energy_total,recorder,kWh,False,True,
-46,sensor.tasmota_energy_today,recorder,kWh,False,True,
-47,sensor.tasmota_energy_power,recorder,W,True,False,
-48,sensor.tasmota_energy_apparentpower,recorder,VA,True,False,
-49,sensor.tasmota_energy_reactivepower,recorder,var,True,False,
-50,sensor.tasmota_energy_factor,recorder,,True,False,
-51,sensor.tasmota_energy_voltage,recorder,V,True,False,
-52,sensor.tasmota_energy_current,recorder,A,True,False,
-53,sensor.tasmota_signal_2,recorder,dBm,True,False,
-54,sensor.tasmota_energy_total_2,recorder,kWh,False,True,
-55,sensor.tasmota_energy_today_2,recorder,kWh,False,True,
-56,sensor.tasmota_energy_power_2,recorder,W,True,False,
-57,sensor.tasmota_energy_apparentpower_2,recorder,VA,True,False,
-58,sensor.tasmota_energy_reactivepower_2,recorder,var,True,False,
-59,sensor.tasmota_energy_factor_2,recorder,,True,False,
-60,sensor.tasmota_energy_voltage_2,recorder,V,True,False,
-61,sensor.tasmota_energy_current_2,recorder,A,True,False,
-62,sensor.backdoor_light_power,recorder,W,True,False,
-63,sensor.backdoor_light_voltage,recorder,V,True,False,
-64,sensor.backdoor_light_current,recorder,A,True,False,
-65,sensor.backdoor_light_energy,recorder,kWh,False,True,
-66,sensor.backdoor_light_device_temperature,recorder,°F,True,False,
-67,sensor.shelly1pm_backlight_rssi,recorder,dBm,True,False

make_import.py

@@ -1,125 +0,0 @@
-import csv
-import pandas as pd
-
-############
-# read files
-############
-
-# file locations
-statistics_meta_archive_file = "input/raw/postgres/statistics_meta.csv"
-statistics_meta_export_file = "input/raw/sqlite/statistics_meta-export.csv"
-statistics_archive_file = "input/raw/postgres/statistics.csv"
-statistics_export_file = "input/raw/sqlite/statistics-export.csv"
-statistics_import_file = "output/statistics-import.csv"
-statistics_short_term_archive_file = "input/raw/postgres/statistics_short_term.csv"
-statistics_short_term_export_file = "input/raw/sqlite/statistics_short_term-export.csv"
-statistics_short_term_import_file = "output/statistics_short_term-import.csv"
-
-# read in current export, and the archive
-meta_df = pd.read_csv(statistics_meta_export_file)
-meta_archive_df = pd.read_csv(statistics_meta_archive_file)
-statistics_df = pd.read_csv(statistics_export_file, index_col='id')
-statistics_archive_df = pd.read_csv(statistics_archive_file, index_col='id')
-statistics_short_term_df = pd.read_csv(statistics_short_term_export_file, index_col='id')
-statistics_short_term_archive_df = pd.read_csv(statistics_short_term_archive_file, index_col='id')
-
-
-#################
-# statistics_meta 
-#################
-
-# find the id's and the unique statistics from each
-meta_df = meta_df[['id','statistic_id']]
-meta_archive_df = meta_archive_df[['id','statistic_id']]
-
-# only keeping statistics from the current system, 
-# form a lookup dictionary so the archive can be corrected
-# using id from the current system export.
-meta_lookup = meta_df.merge(meta_archive_df, on=['statistic_id'], how='left', indicator=True)
-meta_lookup.set_index('id_x').to_csv("meta_merge.csv")
-meta_lookup = meta_lookup[['id_y','id_x']]
-meta_lookup = meta_lookup.T.to_dict('records')[0]
-meta_lookup = dict((v,k) for k,v in meta_lookup.items())
-print(meta_lookup)
-
-############
-# statistics 
-############
-
-# correct the meta column
-statistics_archive_df.replace({'metadata_id': meta_lookup}, inplace=True)
-
-# drop any statistics not in the existing systems metadata
-statistics_archive_df = statistics_archive_df[statistics_archive_df['metadata_id'].isin(meta_lookup.keys())]
-
-# make unique indexes
-statistics_max_id = statistics_df.last_valid_index()
-statistics_archive_df.reset_index(inplace=True)
-statistics_archive_df['id'] += statistics_max_id
-statistics_archive_df.set_index('id',drop=True,inplace=True)
-
-# find any duplicates where tuple (start_ts,metadata_id)
-# exist in export and archive, drop the archive
-# read in current export, and the archive
-print(statistics_archive_df.info())
-statistics_df['unique_tuple'] = statistics_df.apply(lambda row: (row['start_ts'],row['metadata_id']), axis=1) 
-statistics_archive_df['unique_tuple'] = statistics_archive_df.apply(lambda row: (row['start_ts'],row['metadata_id']), axis=1) 
-statistics_archive_df_copy = statistics_archive_df.copy()
-statistics_archive_df_copy = statistics_archive_df_copy[['start_ts','metadata_id','unique_tuple']]
-statistics_df = statistics_df[['start_ts','metadata_id','unique_tuple']]
-unique_lookup = statistics_archive_df_copy.merge(statistics_df, on=['unique_tuple'], how='left', indicator=True)
-unique_lookup = unique_lookup[unique_lookup['_merge']=="both"]
-unique_lookup.to_csv("unique_merge.csv")
-unique_tuples = unique_lookup['unique_tuple']
-statistics_archive_df = statistics_archive_df[~statistics_archive_df['unique_tuple'].isin(unique_tuples)]
-statistics_archive_df.drop(columns='unique_tuple',inplace=True) 
-print(statistics_archive_df.info())
-
-
-
-#######################
-# statistics_short_term 
-#######################
-
-# correct the meta column
-statistics_short_term_archive_df.replace({'metadata_id': meta_lookup}, inplace=True)
-
-# drop any statistics not in the existing systems metadata
-statistics_short_term_archive_df = statistics_short_term_archive_df[statistics_short_term_archive_df['metadata_id'].isin(meta_lookup.keys())]
-
-# make unique indexes
-statistics_short_term_max_id = statistics_short_term_df.last_valid_index()
-statistics_short_term_archive_df.reset_index(inplace=True)
-statistics_short_term_archive_df['id'] += statistics_short_term_max_id
-statistics_short_term_archive_df.set_index('id',drop=True,inplace=True)
-
-# find any duplicates where tuple (start_ts,metadata_id)
-# exist in export and archive, drop the archive
-# read in current export, and the archive
-print(statistics_short_term_archive_df.info())
-statistics_short_term_df['unique_tuple'] = statistics_short_term_df.apply(lambda row: (row['start_ts'],row['metadata_id']), axis=1) 
-statistics_short_term_archive_df['unique_tuple'] = statistics_short_term_archive_df.apply(lambda row: (row['start_ts'],row['metadata_id']), axis=1) 
-statistics_short_term_archive_df_copy = statistics_short_term_archive_df.copy()
-statistics_short_term_archive_df_copy = statistics_short_term_archive_df_copy[['start_ts','metadata_id','unique_tuple']]
-statistics_short_term_df = statistics_short_term_df[['start_ts','metadata_id','unique_tuple']]
-unique_lookup = statistics_short_term_archive_df_copy.merge(statistics_short_term_df, on=['unique_tuple'], how='left', indicator=True)
-#unique_lookup.to_csv(statistics_short_term_import_file)
-unique_lookup = unique_lookup[unique_lookup['_merge']=="both"]
-#unique_lookup.to_csv("unique_merge.csv")
-unique_tuples = unique_lookup['unique_tuple']
-statistics_short_term_archive_df = statistics_short_term_archive_df[~statistics_short_term_archive_df['unique_tuple'].isin(unique_tuples)]
-statistics_short_term_archive_df.drop(columns='unique_tuple',inplace=True) 
-print(statistics_short_term_archive_df.info())
-
-
-
-
-
-###########################
-# write files for importing
-###########################
-
-statistics_archive_df.to_csv(statistics_import_file)
-statistics_short_term_archive_df.to_csv(statistics_short_term_import_file)
-
-

meta_merge.csv

@@ -1,68 +0,0 @@
-id_x,statistic_id,id_y,_merge
-1,sensor.freebsd_router_kib_s_received,2,both
-2,sensor.freebsd_router_kib_s_sent,3,both
-3,sensor.disk_use_percent_config,134,both
-4,sensor.disk_use,135,both
-5,sensor.disk_free,136,both
-6,sensor.memory_use_percent,137,both
-7,sensor.memory_use,138,both
-8,sensor.memory_free,139,both
-9,sensor.swap_use_percent,140,both
-10,sensor.swap_use,141,both
-11,sensor.swap_free,142,both
-12,sensor.load_1m,143,both
-13,sensor.load_5m,144,both
-14,sensor.load_15m,145,both
-15,sensor.network_in_enp0s18,151,both
-16,sensor.network_out_enp0s18,152,both
-17,sensor.network_throughput_in_enp0s18,153,both
-18,sensor.network_throughput_out_enp0s18,154,both
-19,sensor.packets_in_enp0s18,155,both
-20,sensor.packets_out_enp0s18,156,both
-21,sensor.processor_use,146,both
-22,sensor.hvac_cooling_time_yesterday,55,both
-23,sensor.sump_pump_run_counter_today,45,both
-24,sensor.hvac_cooling_time_today,54,both
-25,sensor.hvac_heating_time_yesterday,59,both
-26,sensor.sump_pump_run_counter_yesterday,44,both
-27,sensor.hvac_heating_time_today,58,both
-28,sensor.total_panel_power,35,both
-29,sensor.total_panel_energy,36,both
-30,sensor.total_electric_consumption_cost,41,both
-31,sensor.total_electric_consumption,40,both
-32,sensor.service_leg_02_power,24,both
-33,sensor.service_leg_02_energy,25,both
-34,sensor.service_leg_02_energy_returned,26,both
-35,sensor.service_leg_02_voltage,27,both
-36,sensor.service_leg_01_power,28,both
-37,sensor.service_leg_01_energy,29,both
-38,sensor.service_leg_01_energy_returned,30,both
-39,sensor.service_leg_01_voltage,31,both
-40,sensor.qq8j_battery,56,both
-41,sensor.qq8j_temperature,57,both
-42,sensor.my_ecobee_current_temperature,52,both
-43,sensor.my_ecobee_current_humidity,53,both
-44,sensor.tasmota_signal,13,both
-45,sensor.tasmota_energy_total,5,both
-46,sensor.tasmota_energy_today,6,both
-47,sensor.tasmota_energy_power,7,both
-48,sensor.tasmota_energy_apparentpower,8,both
-49,sensor.tasmota_energy_reactivepower,9,both
-50,sensor.tasmota_energy_factor,10,both
-51,sensor.tasmota_energy_voltage,11,both
-52,sensor.tasmota_energy_current,12,both
-53,sensor.tasmota_signal_2,22,both
-54,sensor.tasmota_energy_total_2,14,both
-55,sensor.tasmota_energy_today_2,15,both
-56,sensor.tasmota_energy_power_2,16,both
-57,sensor.tasmota_energy_apparentpower_2,17,both
-58,sensor.tasmota_energy_reactivepower_2,18,both
-59,sensor.tasmota_energy_factor_2,19,both
-60,sensor.tasmota_energy_voltage_2,20,both
-61,sensor.tasmota_energy_current_2,21,both
-62,sensor.backdoor_light_power,46,both
-63,sensor.backdoor_light_voltage,50,both
-64,sensor.backdoor_light_current,51,both
-65,sensor.backdoor_light_energy,47,both
-66,sensor.backdoor_light_device_temperature,48,both
-67,sensor.shelly1pm_backlight_rssi,49,both

unique_merge.csv

@@ -1 +0,0 @@
-,start_ts_x,metadata_id_x,unique_tuple,start_ts_y,metadata_id_y,_merge