Loading files
This file is pinned to the selected release. Download the original to open it in your design tool.
// SPDX-License-Identifier: GPL-3.0-or-later
// Copyright (C) 2025 Jibril Sharafi
#include "utils.h"
#include "app_image_descriptor.h"
#include "backoff_schedule.h"
#include "custometh.h"
#include "duration_format.h"
#include "sha256_hex.h"
#include "version_compare.h"
#include <esp_random.h>
#include <esp_wifi.h>
#include "taskprofiler.h"
static TaskHandle_t _restartTaskHandle = NULL;
static TaskHandle_t _maintenanceTaskHandle = NULL;
static bool _maintenanceTaskShouldRun = false;
static esp_timer_handle_t _failsafeTimer = NULL;
// A restart request refused by the uptime gate is held here and retried from the
// maintenance loop. Dropping it - which is what returning false alone did - loses the
// request entirely: a WiFi reset issued in the first 30 s of uptime erased the
// credentials, logged "restart delayed" and then never restarted, leaving the device
// running on state that no longer matched what was stored. Deferring keeps the
// anti-restart-loop protection intact; it only stops the request from evaporating.
//
// setRestartSystem() is called from whichever task hits a restart-worthy condition -
// the WiFi task, the HTTP server task, the button task, this module's own maintenance
// task - so these three fields are genuinely shared state and are guarded by a spinlock
// the same way the WiFi module guards its own diagnostic buffers: the section is a
// couple of word writes plus a bounded snprintf, never allocates, never blocks, so a
// portMUX critical section is cheaper and simpler here than a semaphore.
static portMUX_TYPE _pendingRestartMux = portMUX_INITIALIZER_UNLOCKED;
static bool _pendingRestart = false;
static bool _pendingRestartFactoryReset = false;
static char _pendingRestartReason[STATUS_BUFFER_SIZE] = {0};
static TaskHeartbeat _maintenanceHeartbeat;
// Static function declarations
static void _factoryReset();
static void _restartTask(void* parameter);
static void _failsafeRestartCallback(void* parameter);
static void _startFailsafeTimer(uint32_t timeoutMs);
static void _maintenanceTask(void* parameter);
static bool _listLittleFsFilesRecursive(JsonDocument &doc, const char* dirname, const char* basePath, uint8_t levels);
static bool _ensureDirectoryExists(const char* dirPath);
static bool _decompressGzipFile(const char* gzPath, const char* outputPath);
static bool _appendFileToFile(const char* srcPath, File &destFile, bool skipHeader);
// New system info functions
void populateSystemStaticInfo(SystemStaticInfo& info) {
// Initialize the struct to ensure clean state
memset(&info, 0, sizeof(info));
// Product info
snprintf(info.companyName, sizeof(info.companyName), "%s", COMPANY_NAME);
snprintf(info.productName, sizeof(info.productName), "%s", PRODUCT_NAME);
snprintf(info.fullProductName, sizeof(info.fullProductName), "%s", FULL_PRODUCT_NAME);
snprintf(info.productDescription, sizeof(info.productDescription), "%s", PRODUCT_DESCRIPTION);
snprintf(info.githubUrl, sizeof(info.githubUrl), "%s", GITHUB_URL);
snprintf(info.author, sizeof(info.author), "%s", AUTHOR);
snprintf(info.authorEmail, sizeof(info.authorEmail), "%s", AUTHOR_EMAIL);
// Firmware info
snprintf(info.buildVersion, sizeof(info.buildVersion), "%s", FIRMWARE_BUILD_VERSION);
snprintf(info.buildDate, sizeof(info.buildDate), "%s", FIRMWARE_BUILD_DATE);
snprintf(info.buildTime, sizeof(info.buildTime), "%s", FIRMWARE_BUILD_TIME);
snprintf(info.sketchMD5, sizeof(info.sketchMD5), "%s", ESP.getSketchMD5().c_str());
const esp_partition_t *running = esp_ota_get_running_partition();
snprintf(info.partitionAppName, sizeof(info.partitionAppName), "%s", running->label);
// Hardware info
snprintf(info.chipModel, sizeof(info.chipModel), "%s", ESP.getChipModel());
info.chipRevision = ESP.getChipRevision();
info.chipCores = ESP.getChipCores();
info.chipId = ESP.getEfuseMac();
info.flashChipSizeBytes = ESP.getFlashChipSize();
info.flashChipSpeedHz = ESP.getFlashChipSpeed();
info.psramSizeBytes = ESP.getPsramSize();
info.cpuFrequencyMHz = ESP.getCpuFreqMHz();
// Crash and reset monitoring
info.crashCount = CrashMonitor::getCrashCount();
info.consecutiveCrashCount = CrashMonitor::getConsecutiveCrashCount();
info.resetCount = CrashMonitor::getResetCount();
info.consecutiveResetCount = CrashMonitor::getConsecutiveResetCount();
info.lastResetReason = (uint32_t)esp_reset_reason();
snprintf(info.lastResetReasonString, sizeof(info.lastResetReasonString), "%s", getResetReasonString(esp_reset_reason()));
info.lastResetWasCrash = CrashMonitor::isLastResetDueToCrash();
// SDK info
snprintf(info.sdkVersion, sizeof(info.sdkVersion), "%s", ESP.getSdkVersion());
snprintf(info.coreVersion, sizeof(info.coreVersion), "%s", ESP.getCoreVersion());
// Device ID
getDeviceId(info.deviceId, sizeof(info.deviceId));
// Factory provisioning data (written at manufacturing time; defaults kept otherwise)
Preferences factoryPrefs;
if (factoryPrefs.begin(PREFERENCES_NAMESPACE_FACTORY, true)) {
factoryPrefs.getString(FACTORY_KEY_SERIAL_NUMBER, info.serialNumber, sizeof(info.serialNumber));
if (strlen(info.serialNumber) == 0) {
snprintf(info.serialNumber, sizeof(info.serialNumber), "Unknown");
}
info.manufacturingUnixTs = factoryPrefs.getULong(FACTORY_KEY_MFG_TS, 0);
factoryPrefs.end();
}
// Selected hardware profile (authoritative post-selection, works in community mode too)
snprintf(info.productLine, sizeof(info.productLine), "%s", productLineToString(globalHwProfile->product));
snprintf(info.pcbRevision, sizeof(info.pcbRevision), "v%u.%u",
(unsigned)(globalHwProfile->version / 10), (unsigned)(globalHwProfile->version % 10));
info.communityMode = globalCommunityMode;
// Running image's own descriptor
info.imgDescPsramMb = ENERGYME_APP_DESC.psramMb;
snprintf(info.imgDescFwVersion, sizeof(info.imgDescFwVersion), "%s", ENERGYME_APP_DESC.fwVersion);
snprintf(info.imgDescBuildEnv, sizeof(info.imgDescBuildEnv), "%s", ENERGYME_APP_DESC.buildEnv);
snprintf(info.imgDescGitRev, sizeof(info.imgDescGitRev), "%s", ENERGYME_APP_DESC.gitRev);
info.imgDescMinPcbVersion = ENERGYME_APP_DESC.minPcbVersion;
info.imgDescMaxPcbVersion = ENERGYME_APP_DESC.maxPcbVersion;
info.imgDescPartitionLayoutId = ENERGYME_APP_DESC.partitionLayoutId;
// Passive/"other" OTA partition's descriptor, if it has a valid one
ImageDescriptor::Descriptor otherDesc;
info.otherImgDescPresent = getOtherPartitionImageDescriptor(otherDesc);
if (info.otherImgDescPresent) {
snprintf(info.otherImgDescProduct, sizeof(info.otherImgDescProduct), "%s", otherDesc.product);
info.otherImgDescPsramMb = otherDesc.psramMb;
snprintf(info.otherImgDescFwVersion, sizeof(info.otherImgDescFwVersion), "%s", otherDesc.fwVersion);
snprintf(info.otherImgDescBuildEnv, sizeof(info.otherImgDescBuildEnv), "%s", otherDesc.buildEnv);
snprintf(info.otherImgDescGitRev, sizeof(info.otherImgDescGitRev), "%s", otherDesc.gitRev);
info.otherImgDescMinPcbVersion = otherDesc.minPcbVersion;
info.otherImgDescMaxPcbVersion = otherDesc.maxPcbVersion;
info.otherImgDescPartitionLayoutId = otherDesc.partitionLayoutId;
}
LOG_DEBUG("Static system info populated");
}
void populateSystemDynamicInfo(SystemDynamicInfo& info) {
// Initialize the struct to ensure clean state
memset(&info, 0, sizeof(info));
// Time
info.uptimeMilliseconds = millis64();
info.uptimeSeconds = info.uptimeMilliseconds / 1000;
CustomTime::getTimestampIso(info.currentTimestampIso, sizeof(info.currentTimestampIso));
// Memory - Heap
info.heapTotalBytes = ESP.getHeapSize();
info.heapFreeBytes = ESP.getFreeHeap();
info.heapUsedBytes = info.heapTotalBytes - info.heapFreeBytes;
info.heapMinFreeBytes = ESP.getMinFreeHeap();
info.heapMaxAllocBytes = ESP.getMaxAllocHeap();
info.heapFreePercentage = info.heapTotalBytes > 0 ? ((float)info.heapFreeBytes / (float)info.heapTotalBytes) * 100.0f : 0.0f;
info.heapUsedPercentage = 100.0f - info.heapFreePercentage;
// Memory - PSRAM
info.psramTotalBytes = ESP.getPsramSize();
if (info.psramTotalBytes > 0) {
info.psramFreeBytes = ESP.getFreePsram();
info.psramUsedBytes = info.psramTotalBytes - info.psramFreeBytes;
info.psramMinFreeBytes = ESP.getMinFreePsram();
info.psramMaxAllocBytes = ESP.getMaxAllocPsram();
info.psramFreePercentage = info.psramTotalBytes > 0 ? ((float)info.psramFreeBytes / (float)info.psramTotalBytes) * 100.0f : 0.0f;
info.psramUsedPercentage = 100.0f - info.psramFreePercentage;
} else {
info.psramFreeBytes = 0;
info.psramUsedBytes = 0;
info.psramMinFreeBytes = 0;
info.psramMaxAllocBytes = 0;
info.psramFreePercentage = 0.0f;
info.psramUsedPercentage = 0.0f;
}
// Storage - LittleFS
info.littlefsTotalBytes = LittleFS.totalBytes();
info.littlefsUsedBytes = LittleFS.usedBytes();
info.littlefsFreeBytes = info.littlefsTotalBytes - info.littlefsUsedBytes;
info.littlefsFreePercentage = info.littlefsTotalBytes > 0 ? ((float)info.littlefsFreeBytes / (float)info.littlefsTotalBytes) * 100.0f : 0.0f;
info.littlefsUsedPercentage = 100.0f - info.littlefsFreePercentage;
// Storage - NVS
nvs_stats_t nvs_stats;
esp_err_t err = nvs_get_stats(NULL, &nvs_stats);
if (err != ESP_OK) {
LOG_ERROR("Failed to get NVS stats: %s", esp_err_to_name(err));
info.usedEntries = 0;
info.availableEntries = 0;
info.totalUsableEntries = 0;
info.usedEntriesPercentage = 0.0f;
info.availableEntriesPercentage = 0.0f;
info.namespaceCount = 0;
} else {
info.usedEntries = nvs_stats.used_entries;
info.availableEntries = nvs_stats.available_entries;
info.totalUsableEntries = info.usedEntries + info.availableEntries; // Some are reserved
info.usedEntriesPercentage = info.totalUsableEntries > 0 ? ((float)info.usedEntries / (float)info.totalUsableEntries) * 100.0f : 0.0f;
info.availableEntriesPercentage = info.totalUsableEntries > 0 ? ((float)info.availableEntries / (float)info.totalUsableEntries) * 100.0f : 0.0f;
info.namespaceCount = nvs_stats.namespace_count;
}
// Performance
info.temperatureCelsius = temperatureRead();
// CPU stats over a window matching the system_dynamic publish cadence.
// ENV_DEV: last 60 s; ENV_PROD: full 3600 s ring.
#ifdef ENV_DEV
static constexpr uint32_t CPU_STATS_WINDOW_SECONDS = 60;
#else
static constexpr uint32_t CPU_STATS_WINDOW_SECONDS = 0; // 0 = full ring
#endif
info.cpuCore0Stats = TaskProfiler::getCpuStats(0, CPU_STATS_WINDOW_SECONDS);
info.cpuCore1Stats = TaskProfiler::getCpuStats(1, CPU_STATS_WINDOW_SECONDS);
// Network (if connected)
if (CustomWifi::isFullyConnected()) {
info.wifiConnected = true;
info.wifiRssi = WiFi.RSSI();
snprintf(info.wifiSsid, sizeof(info.wifiSsid), "%s", WiFi.SSID().c_str());
snprintf(info.wifiLocalIp, sizeof(info.wifiLocalIp), "%s", WiFi.localIP().toString().c_str());
snprintf(info.wifiGatewayIp, sizeof(info.wifiGatewayIp), "%s", WiFi.gatewayIP().toString().c_str());
snprintf(info.wifiSubnetMask, sizeof(info.wifiSubnetMask), "%s", WiFi.subnetMask().toString().c_str());
snprintf(info.wifiDnsIp, sizeof(info.wifiDnsIp), "%s", WiFi.dnsIP().toString().c_str());
snprintf(info.wifiBssid, sizeof(info.wifiBssid), "%s", WiFi.BSSIDstr().c_str());
} else {
info.wifiConnected = false;
info.wifiRssi = -100; // Invalid RSSI
snprintf(info.wifiSsid, sizeof(info.wifiSsid), "Not connected");
snprintf(info.wifiLocalIp, sizeof(info.wifiLocalIp), "0.0.0.0");
snprintf(info.wifiGatewayIp, sizeof(info.wifiGatewayIp), "0.0.0.0");
snprintf(info.wifiSubnetMask, sizeof(info.wifiSubnetMask), "0.0.0.0");
snprintf(info.wifiDnsIp, sizeof(info.wifiDnsIp), "0.0.0.0");
snprintf(info.wifiBssid, sizeof(info.wifiBssid), "00:00:00:00:00:00");
}
snprintf(info.wifiMacAddress, sizeof(info.wifiMacAddress), "%s", WiFi.macAddress().c_str()); // MAC is available even when disconnected
// Ethernet. Every field is written on every path: printDeviceStatusDynamic() hands in a
// raw ps_malloc() block, so the struct constructor never ran. Arbitration never runs on
// products without Ethernet, so the active interface is derived from the WiFi state there.
info.ethEnabled = CustomEth::isEnabled();
info.ethLinkUp = info.ethEnabled && CustomEth::isLinkUp();
info.ethLinkSpeedMbps = info.ethLinkUp ? ETH.linkSpeed() : 0;
info.ethFullDuplex = info.ethLinkUp && ETH.fullDuplex();
if (info.ethEnabled) {
snprintf(info.ethLocalIp, sizeof(info.ethLocalIp), "%s", ETH.localIP().toString().c_str());
snprintf(info.ethMacAddress, sizeof(info.ethMacAddress), "%s", ETH.macAddress().c_str());
snprintf(info.activeInterface, sizeof(info.activeInterface), "%s",
InterfaceArbitration::interfaceName(CustomEth::activeInterface()));
} else {
snprintf(info.ethLocalIp, sizeof(info.ethLocalIp), "0.0.0.0");
snprintf(info.ethMacAddress, sizeof(info.ethMacAddress), "00:00:00:00:00:00");
snprintf(info.activeInterface, sizeof(info.activeInterface), "%s",
InterfaceArbitration::interfaceName(info.wifiConnected ? InterfaceArbitration::Interface::WIFI_STATION
: InterfaceArbitration::Interface::NONE));
}
// Tasks
if (!globalCommunityMode) {
info.mqttTaskInfo = Mqtt::getMqttTaskInfo();
info.mqttOtaTaskInfo = Mqtt::getMqttOtaTaskInfo();
}
info.customMqttTaskInfo = CustomMqtt::getTaskInfo();
info.customServerHealthCheckTaskInfo = CustomServer::getHealthCheckTaskInfo();
info.customServerOtaTimeoutTaskInfo = CustomServer::getOtaTimeoutTaskInfo();
info.ledTaskInfo = Led::getTaskInfo();
info.influxDbTaskInfo = InfluxDbClient::getTaskInfo();
info.crashMonitorTaskInfo = CrashMonitor::getTaskInfo();
info.buttonHandlerTaskInfo = ButtonHandler::getTaskInfo();
info.udpLogTaskInfo = CustomLog::getTaskInfo();
info.customWifiTaskInfo = CustomWifi::getTaskInfo();
if (globalHwProfile->hasEthernet) info.customEthTaskInfo = CustomEth::getTaskInfo();
info.ade7953MeterReadingTaskInfo = Ade7953::getMeterReadingTaskInfo();
info.ade7953EnergySaveTaskInfo = Ade7953::getEnergySaveTaskInfo();
info.ade7953HourlyCsvTaskInfo = Ade7953::getHourlyCsvTaskInfo();
info.ade7953GridSamplerTaskInfo = Ade7953::getGridSamplerTaskInfo();
info.maintenanceTaskInfo = getMaintenanceTaskInfo();
info.issueRegistryTaskInfo = IssueRegistry::getTaskInfo();
LOG_DEBUG("Dynamic system info populated");
}
void systemStaticInfoToJson(SystemStaticInfo& info, JsonDocument &doc) {
// No need to use String for dandling pointer data since most of it here is
// coming from compiled static variables
// Product
doc["product"]["companyName"] = info.companyName;
doc["product"]["productName"] = info.productName;
doc["product"]["fullProductName"] = info.fullProductName;
doc["product"]["productDescription"] = info.productDescription;
doc["product"]["githubUrl"] = info.githubUrl;
doc["product"]["author"] = info.author;
doc["product"]["authorEmail"] = info.authorEmail;
// Firmware
doc["firmware"]["buildVersion"] = info.buildVersion;
doc["firmware"]["buildDate"] = info.buildDate;
doc["firmware"]["buildTime"] = info.buildTime;
doc["firmware"]["sketchMD5"] = info.sketchMD5;
doc["firmware"]["partitionAppName"] = info.partitionAppName;
// Hardware
doc["hardware"]["chipModel"] = info.chipModel;
doc["hardware"]["chipRevision"] = info.chipRevision;
doc["hardware"]["chipCores"] = info.chipCores;
doc["hardware"]["chipId"] = (uint64_t)info.chipId;
doc["hardware"]["cpuFrequencyMHz"] = info.cpuFrequencyMHz;
doc["hardware"]["flashChipSizeBytes"] = info.flashChipSizeBytes;
doc["hardware"]["flashChipSpeedHz"] = info.flashChipSpeedHz;
doc["hardware"]["psramSizeBytes"] = info.psramSizeBytes;
doc["hardware"]["cpuFrequencyMHz"] = info.cpuFrequencyMHz;
// Crash monitoring
doc["monitoring"]["crashCount"] = info.crashCount;
doc["monitoring"]["consecutiveCrashCount"] = info.consecutiveCrashCount;
doc["monitoring"]["resetCount"] = info.resetCount;
doc["monitoring"]["consecutiveResetCount"] = info.consecutiveResetCount;
doc["monitoring"]["lastResetReason"] = info.lastResetReason;
doc["monitoring"]["lastResetReasonString"] = info.lastResetReasonString;
doc["monitoring"]["lastResetWasCrash"] = info.lastResetWasCrash;
// SDK
doc["sdk"]["sdkVersion"] = info.sdkVersion;
doc["sdk"]["coreVersion"] = info.coreVersion;
// Device
doc["device"]["id"] = info.deviceId;
#ifdef ENV_DEV
doc["device"]["devBuild"] = true;
#else
doc["device"]["devBuild"] = false;
#endif
// Factory provisioning (Unknown / 0 when the device was not factory-provisioned)
doc["factory"]["serialNumber"] = info.serialNumber;
doc["factory"]["manufacturingUnixTs"] = info.manufacturingUnixTs;
doc["factory"]["productLine"] = info.productLine;
doc["factory"]["pcbRevision"] = info.pcbRevision;
doc["factory"]["communityMode"] = info.communityMode;
// Image descriptor (see lib/image_descriptor)
doc["imageDescriptor"]["running"]["psramMb"] = info.imgDescPsramMb;
doc["imageDescriptor"]["running"]["fwVersion"] = info.imgDescFwVersion;
doc["imageDescriptor"]["running"]["buildEnv"] = info.imgDescBuildEnv;
doc["imageDescriptor"]["running"]["gitRev"] = info.imgDescGitRev;
doc["imageDescriptor"]["running"]["minPcbVersion"] = info.imgDescMinPcbVersion;
doc["imageDescriptor"]["running"]["maxPcbVersion"] = info.imgDescMaxPcbVersion;
doc["imageDescriptor"]["running"]["partitionLayoutId"] = info.imgDescPartitionLayoutId;
doc["imageDescriptor"]["other"]["present"] = info.otherImgDescPresent;
if (info.otherImgDescPresent) {
doc["imageDescriptor"]["other"]["product"] = info.otherImgDescProduct;
doc["imageDescriptor"]["other"]["psramMb"] = info.otherImgDescPsramMb;
doc["imageDescriptor"]["other"]["fwVersion"] = info.otherImgDescFwVersion;
doc["imageDescriptor"]["other"]["buildEnv"] = info.otherImgDescBuildEnv;
doc["imageDescriptor"]["other"]["gitRev"] = info.otherImgDescGitRev;
doc["imageDescriptor"]["other"]["minPcbVersion"] = info.otherImgDescMinPcbVersion;
doc["imageDescriptor"]["other"]["maxPcbVersion"] = info.otherImgDescMaxPcbVersion;
doc["imageDescriptor"]["other"]["partitionLayoutId"] = info.otherImgDescPartitionLayoutId;
}
LOG_DEBUG("Static system info converted to JSON");
}
void systemDynamicInfoToJson(SystemDynamicInfo& info, JsonDocument &doc) {
// Time
doc["time"]["uptimeMilliseconds"] = (uint64_t)info.uptimeMilliseconds;
doc["time"]["uptimeSeconds"] = info.uptimeSeconds;
doc["time"]["currentTimestampIso"] = info.currentTimestampIso;
// Memory - Heap
doc["memory"]["heap"]["totalBytes"] = info.heapTotalBytes;
doc["memory"]["heap"]["freeBytes"] = info.heapFreeBytes;
doc["memory"]["heap"]["usedBytes"] = info.heapUsedBytes;
doc["memory"]["heap"]["minFreeBytes"] = info.heapMinFreeBytes;
doc["memory"]["heap"]["maxAllocBytes"] = info.heapMaxAllocBytes;
doc["memory"]["heap"]["freePercentage"] = info.heapFreePercentage;
doc["memory"]["heap"]["usedPercentage"] = info.heapUsedPercentage;
// Memory - PSRAM
doc["memory"]["psram"]["totalBytes"] = info.psramTotalBytes;
doc["memory"]["psram"]["freeBytes"] = info.psramFreeBytes;
doc["memory"]["psram"]["usedBytes"] = info.psramUsedBytes;
doc["memory"]["psram"]["minFreeBytes"] = info.psramMinFreeBytes;
doc["memory"]["psram"]["maxAllocBytes"] = info.psramMaxAllocBytes;
doc["memory"]["psram"]["freePercentage"] = info.psramFreePercentage;
doc["memory"]["psram"]["usedPercentage"] = info.psramUsedPercentage;
// Storage
doc["storage"]["littlefs"]["totalBytes"] = info.littlefsTotalBytes;
doc["storage"]["littlefs"]["usedBytes"] = info.littlefsUsedBytes;
doc["storage"]["littlefs"]["freeBytes"] = info.littlefsFreeBytes;
doc["storage"]["littlefs"]["freePercentage"] = info.littlefsFreePercentage;
doc["storage"]["littlefs"]["usedPercentage"] = info.littlefsUsedPercentage;
// Storage - NVS
doc["storage"]["nvs"]["totalUsableEntries"] = info.totalUsableEntries;
doc["storage"]["nvs"]["usedEntries"] = info.usedEntries;
doc["storage"]["nvs"]["availableEntries"] = info.availableEntries;
doc["storage"]["nvs"]["usedEntriesPercentage"] = info.usedEntriesPercentage;
doc["storage"]["nvs"]["availableEntriesPercentage"] = info.availableEntriesPercentage;
doc["storage"]["nvs"]["namespaceCount"] = info.namespaceCount;
// Performance
doc["performance"]["temperatureCelsius"] = info.temperatureCelsius;
// CPU stats (per core, spike-aware ring buffer)
auto addCpuCore = [&](const char* key, const TaskProfiler::CpuStats& s) {
doc["cpu"][key]["current"] = s.current;
doc["cpu"][key]["min"] = s.min;
doc["cpu"][key]["max"] = s.max;
doc["cpu"][key]["avg"] = s.avg;
doc["cpu"][key]["p50"] = s.p50;
doc["cpu"][key]["p90"] = s.p90;
doc["cpu"][key]["p95"] = s.p95;
doc["cpu"][key]["p99"] = s.p99;
doc["cpu"][key]["samplesInWindow"] = s.samplesInWindow;
doc["cpu"][key]["samplesAboveThreshold"] = s.samplesAboveThreshold;
doc["cpu"][key]["windowSeconds"] = s.windowSeconds;
};
addCpuCore("core0", info.cpuCore0Stats);
addCpuCore("core1", info.cpuCore1Stats);
// Network
doc["network"]["wifiConnected"] = info.wifiConnected;
doc["network"]["wifiSsid"] = JsonString(info.wifiSsid); // Ensure it is not a dangling pointer
doc["network"]["wifiMacAddress"] = JsonString(info.wifiMacAddress); // Ensure it is not a dangling pointer
doc["network"]["wifiLocalIp"] = JsonString(info.wifiLocalIp); // Ensure it is not a dangling pointer
doc["network"]["wifiGatewayIp"] = JsonString(info.wifiGatewayIp); // Ensure it is not a dangling pointer
doc["network"]["wifiSubnetMask"] = JsonString(info.wifiSubnetMask); // Ensure it is not a dangling pointer
doc["network"]["wifiDnsIp"] = JsonString(info.wifiDnsIp); // Ensure it is not a dangling pointer
doc["network"]["wifiBssid"] = JsonString(info.wifiBssid); // Ensure it is not a dangling pointer
doc["network"]["wifiRssi"] = info.wifiRssi;
doc["network"]["activeInterface"] = JsonString(info.activeInterface); // Ensure it is not a dangling pointer
doc["network"]["ethEnabled"] = info.ethEnabled;
if (info.ethEnabled) {
doc["network"]["ethLinkUp"] = info.ethLinkUp;
doc["network"]["ethLocalIp"] = JsonString(info.ethLocalIp); // Ensure it is not a dangling pointer
doc["network"]["ethMacAddress"] = JsonString(info.ethMacAddress); // Ensure it is not a dangling pointer
doc["network"]["ethLinkSpeedMbps"] = info.ethLinkSpeedMbps;
doc["network"]["ethFullDuplex"] = info.ethFullDuplex;
}
// Tasks
uint64_t nowMs = millis64();
auto addTask = [&](const char* name, const TaskInfo& t) {
doc["tasks"][name]["allocatedStack"] = t.allocatedStack;
doc["tasks"][name]["minimumFreeStack"] = t.minimumFreeStack;
doc["tasks"][name]["freePercentage"] = t.freePercentage;
doc["tasks"][name]["usedPercentage"] = t.usedPercentage;
doc["tasks"][name]["lastTickMillis"] = (uint64_t)t.lastTickMillis;
doc["tasks"][name]["loopCount"] = t.loopCount;
// staleMs == 0 means "task not instrumented" (no TASK_HEARTBEAT call) - it does
// not mean the task ticked this exact ms. Consumers that care about liveness
// should also check loopCount > 0.
doc["tasks"][name]["staleMs"] = (t.lastTickMillis == 0) ? 0ULL : (nowMs - t.lastTickMillis);
};
addTask("mqtt", info.mqttTaskInfo);
addTask("mqttOta", info.mqttOtaTaskInfo);
addTask("customMqtt", info.customMqttTaskInfo);
addTask("customServerHealthCheck", info.customServerHealthCheckTaskInfo);
addTask("customServerOtaTimeout", info.customServerOtaTimeoutTaskInfo);
addTask("led", info.ledTaskInfo);
addTask("influxDb", info.influxDbTaskInfo);
addTask("crashMonitor", info.crashMonitorTaskInfo);
addTask("buttonHandler", info.buttonHandlerTaskInfo);
addTask("udpLog", info.udpLogTaskInfo);
addTask("customWifi", info.customWifiTaskInfo);
if (globalHwProfile->hasEthernet) addTask("customEth", info.customEthTaskInfo);
addTask("ade7953MeterReading", info.ade7953MeterReadingTaskInfo);
addTask("ade7953EnergySave", info.ade7953EnergySaveTaskInfo);
addTask("ade7953HourlyCsv", info.ade7953HourlyCsvTaskInfo);
addTask("ade7953GridSampler", info.ade7953GridSamplerTaskInfo);
addTask("maintenance", info.maintenanceTaskInfo);
addTask("issueRegistry", info.issueRegistryTaskInfo);
// Tasks the firmware does not own (libraries, core, IDF): only the stack low-water mark is
// known. Looked up by name, so a task that does not exist on this product is left out.
static const char *const systemTaskNames[] = {"AdvancedLogTask", "async_tcp", "arduino_events", "tiT",
"sys_evt", "esp_timer", "wifi", "w5500_tsk", "ipc0", "ipc1"};
for (const char *name : systemTaskNames) {
TaskHandle_t handle = xTaskGetHandle(name);
if (handle != NULL) doc["systemTasks"][name]["minimumFreeStack"] = (uint32_t)uxTaskGetStackHighWaterMark(handle);
}
LOG_DEBUG("Dynamic system info converted to JSON");
}
void getJsonDeviceStaticInfo(JsonDocument &doc) {
SystemStaticInfo info;
populateSystemStaticInfo(info);
systemStaticInfoToJson(info, doc);
}
void getJsonDeviceDynamicInfo(JsonDocument &doc) {
SystemDynamicInfo info;
populateSystemDynamicInfo(info);
systemDynamicInfoToJson(info, doc);
}
bool safeSerializeJson(JsonDocument &jsonDocument, char* buffer, size_t bufferSize, bool truncateOnError) {
// Validate inputs
if (!buffer || bufferSize == 0) {
LOG_WARNING("Invalid buffer parameters passed to safeSerializeJson");
return false;
}
size_t size = measureJson(jsonDocument);
if (size >= bufferSize) {
if (truncateOnError) {
// Truncate JSON to fit buffer
serializeJson(jsonDocument, buffer, bufferSize);
// Ensure null-termination (avoid weird last character issues)
buffer[bufferSize - 1] = '\0';
LOG_DEBUG("Truncating JSON to fit buffer size (%zu bytes vs %zu bytes)", bufferSize, size);
} else {
LOG_WARNING("JSON size (%zu bytes) exceeds buffer size (%zu bytes)", size, bufferSize);
snprintf(buffer, bufferSize, "%s", ""); // Clear buffer on failure
}
return false;
}
serializeJson(jsonDocument, buffer, bufferSize);
LOG_VERBOSE("JSON serialized successfully (bytes: %zu): %s", size, buffer);
return true;
}
// Task function that handles periodic maintenance checks
static void _maintenanceTask(void* parameter) {
LOG_DEBUG("Maintenance task started");
_maintenanceTaskShouldRun = true;
while (_maintenanceTaskShouldRun) {
TASK_HEARTBEAT(_maintenanceHeartbeat);
// Retry a restart the uptime gate refused earlier, now that it may be allowed.
// Snapshot under the lock, then act outside it - setRestartSystem() logs and can
// create a task, neither of which belongs inside a critical section.
bool retryPending = false;
char reason[STATUS_BUFFER_SIZE];
bool factoryReset = false;
taskENTER_CRITICAL(&_pendingRestartMux);
if (_pendingRestart && CrashMonitor::canRestartNow()) {
retryPending = true;
snprintf(reason, sizeof(reason), "%s", _pendingRestartReason);
factoryReset = _pendingRestartFactoryReset;
_pendingRestart = false;
}
taskEXIT_CRITICAL(&_pendingRestartMux);
if (retryPending) {
LOG_INFO("Retrying restart that was delayed by the uptime gate");
setRestartSystem(reason, factoryReset);
}
// Update and print statistics
printStatistics();
printDeviceStatusDynamic();
// Check heap memory
if (ESP.getFreeHeap() < MINIMUM_FREE_HEAP_SIZE) {
LOG_FATAL("Heap memory has degraded below safe minimum (%d bytes): %lu bytes", MINIMUM_FREE_HEAP_SIZE, ESP.getFreeHeap());
setRestartSystem("Heap memory has degraded below safe minimum");
}
// Check PSRAM memory
if (ESP.getFreePsram() < MINIMUM_FREE_PSRAM_SIZE) {
LOG_FATAL("PSRAM memory has degraded below safe minimum (%d bytes): %lu bytes", MINIMUM_FREE_PSRAM_SIZE, ESP.getFreePsram());
setRestartSystem("PSRAM memory has degraded below safe minimum");
}
// Check LittleFS memory and clear log if needed
if (LittleFS.totalBytes() - LittleFS.usedBytes() < MINIMUM_FREE_LITTLEFS_SIZE) {
AdvancedLogger::clearLog(); // Here we clear all for safety
LOG_WARNING("Log cleared due to low LittleFS memory");
}
LOG_DEBUG("Maintenance checks completed");
// Wait for stop notification with timeout (blocking)
if (ulTaskNotifyTake(pdTRUE, pdMS_TO_TICKS(MAINTENANCE_CHECK_INTERVAL)) > 0) {
_maintenanceTaskShouldRun = false;
break;
}
}
LOG_DEBUG("Maintenance task stopping");
_maintenanceTaskHandle = NULL;
vTaskDelete(NULL);
}
void startMaintenanceTask() {
if (_maintenanceTaskHandle != NULL) {
LOG_DEBUG("Maintenance task is already running");
return;
}
LOG_DEBUG("Starting maintenance task with %d bytes stack", TASK_MAINTENANCE_STACK_SIZE);
BaseType_t result = xTaskCreate(
_maintenanceTask,
TASK_MAINTENANCE_NAME,
TASK_MAINTENANCE_STACK_SIZE,
NULL,
TASK_MAINTENANCE_PRIORITY,
&_maintenanceTaskHandle);
if (result != pdPASS) {
LOG_ERROR("Failed to create maintenance task");
}
}
void stopTaskGracefully(TaskHandle_t* taskHandle, const char* taskName) {
if (!taskHandle || *taskHandle == NULL) {
LOG_DEBUG("%s was not running", taskName ? taskName : "Task");
return;
}
LOG_DEBUG("Stopping %s...", taskName ? taskName : "task");
xTaskNotify(*taskHandle, TASK_NOTIFY_SHUTDOWN_BIT, eSetBits);
// Wait with timeout for clean shutdown
int32_t timeout = TASK_STOPPING_TIMEOUT;
uint32_t loops = 0;
while (*taskHandle != NULL && timeout > 0 && loops < MAX_LOOP_ITERATIONS) {
loops++;
delay(TASK_STOPPING_CHECK_INTERVAL);
timeout -= TASK_STOPPING_CHECK_INTERVAL;
}
// Force cleanup if needed
if (*taskHandle != NULL) {
LOG_WARNING("Force stopping %s", taskName ? taskName : "task");
vTaskDelete(*taskHandle);
*taskHandle = NULL;
} else {
LOG_DEBUG("%s stopped successfully", taskName ? taskName : "Task");
}
}
void stopMaintenanceTask() {
stopTaskGracefully(&_maintenanceTaskHandle, "maintenance task");
}
TaskInfo getMaintenanceTaskInfo()
{
return getTaskInfoSafely(_maintenanceTaskHandle, TASK_MAINTENANCE_STACK_SIZE, &_maintenanceHeartbeat);
}
// Failsafe timer callback - runs from esp_timer task context, guaranteed to execute
// even if all other tasks are blocked/deadlocked
static void _failsafeRestartCallback(void* parameter) {
// Don't log here - we're in timer context and logging might be what's blocked
ESP.restart();
}
// Start the failsafe timer that guarantees restart even if something blocks
static void _startFailsafeTimer(uint32_t timeoutMs) {
// Ensure only one timer is running
if (_failsafeTimer != NULL) return;
const esp_timer_create_args_t timerArgs = {
.callback = _failsafeRestartCallback,
.arg = NULL,
.dispatch_method = ESP_TIMER_TASK,
.name = SYSTEM_RESTART_FAILSAFE_TIMER_NAME,
.skip_unhandled_events = true
};
if (esp_timer_create(&timerArgs, &_failsafeTimer) == ESP_OK) {
esp_timer_start_once(_failsafeTimer, timeoutMs * 1000ULL); // Convert ms to us
char failsafeHuman[DURATION_FORMAT_BUFFER_SIZE];
DurationFormat::humanizeDuration((uint64_t)timeoutMs, failsafeHuman, sizeof(failsafeHuman));
LOG_DEBUG("Failsafe restart timer started (%s)", failsafeHuman);
} else {
LOG_WARNING("Failed to create failsafe timer - restart may hang if blocked");
}
}
// Restart task - handles service shutdown and restart (failsafe timer protects this from blocking)
static void _restartTask(void* parameter) {
bool factoryReset = (bool)(uintptr_t)parameter;
LOG_DEBUG("Restart task started%s", factoryReset ? " (factory reset)" : "");
// 1. Visual indicator
Led::setOrange(Led::PRIO_CRITICAL);
// 2. Stop all services (best effort, don't wait forever for each)
LOG_DEBUG("Stopping services before restart...");
IssueRegistry::stop();
CustomServer::stop();
Ade7953::stop();
ModbusTcp::stop();
if (!globalCommunityMode) Mqtt::stop();
LOG_DEBUG("Services stopped");
// 3. Close log file (if this blocks, failsafe timer will restart us)
AdvancedLogger::end();
// 4. Factory reset if requested
LOG_INFO("Restarting system%s", factoryReset ? ". Factory reset requested" : "");
if (factoryReset) { _factoryReset(); }
// 5. Normal restart - if we get here, great. If not, failsafe timer handles it.
ESP.restart();
// Should never reach here
vTaskDelete(NULL);
}
bool setRestartSystem(const char* reason, bool factoryReset) {
LOG_INFO("Restart required for reason: %s%s", reason, factoryReset ? ". Factory reset required" : "");
// Check if we can restart now (safe mode protection)
if (!CrashMonitor::canRestartNow()) {
uint32_t remainingMs = CrashMonitor::getMinimumUptimeRemaining();
uint32_t remainingSec = remainingMs / 1000;
if (CrashMonitor::isInSafeMode()) {
LOG_FATAL(
"SAFE MODE: Restart blocked for %lu s to prevent infinite loops. "
"Reason: %s. WiFi/OTA active for remote recovery",
remainingSec, reason
);
} else {
LOG_WARNING(
"Restart delayed: minimum uptime not reached (%lu s remaining to "
"prevent rapid restart loops)",
remainingSec
);
}
// Hold the request rather than discarding it. The gate exists to space restarts
// out, not to cancel them, and a caller that has already changed persistent state
// (resetWifi() erases the credentials before asking) cannot undo its half of the
// operation when this returns false. First request wins: a later one would only
// overwrite the reason for the same restart. setRestartSystem() can be called
// concurrently from several tasks, so this is guarded the same way as the read.
taskENTER_CRITICAL(&_pendingRestartMux);
if (!_pendingRestart) {
_pendingRestart = true;
_pendingRestartFactoryReset = factoryReset;
snprintf(_pendingRestartReason, sizeof(_pendingRestartReason), "%s", reason);
} else if (factoryReset && !_pendingRestartFactoryReset) {
// A factory reset is the stronger request and must not be masked by a plain
// restart that happened to be asked for first.
_pendingRestartFactoryReset = true;
snprintf(_pendingRestartReason, sizeof(_pendingRestartReason), "%s", reason);
}
taskEXIT_CRITICAL(&_pendingRestartMux);
return false;
}
if (_restartTaskHandle != NULL) {
LOG_INFO("A restart is already scheduled. Keeping the existing one.");
return false;
}
// START FAILSAFE TIMER FIRST - guarantees restart even if task creation or execution fails
_startFailsafeTimer(factoryReset ? SYSTEM_FACTORY_RESET_FAILSAFE_TIMEOUT : SYSTEM_RESTART_FAILSAFE_TIMEOUT);
// Create task to handle graceful shutdown and restart
BaseType_t result = xTaskCreate(
_restartTask,
TASK_RESTART_NAME,
TASK_RESTART_STACK_SIZE,
(void*)(uintptr_t)factoryReset,
TASK_RESTART_PRIORITY,
&_restartTaskHandle);
if (result != pdPASS) {
// Task creation failed, but failsafe timer is already running
LOG_ERROR("Failed to create restart task - failsafe timer will restart system");
}
return true;
}
// Firmware rollback (#237)
// -----------------------------
static_assert(SHA256_HEX_BUFFER_SIZE == Sha256Hex::BUFFER_SIZE,
"sha256 hex buffers are sized by constants.h but written by lib/sha256_hex");
// Forwarder - the logic lives in lib/sha256_hex so it is host-testable
void sha256BytesToHex(const uint8_t sha256[32], char* out, size_t outSize) {
Sha256Hex::bytesToHex(sha256, out, outSize);
}
static const esp_partition_t* _getPassiveOtaPartition() {
const esp_partition_t* running = esp_ota_get_running_partition();
const esp_partition_t* passive = esp_ota_get_next_update_partition(NULL);
if (!passive || passive == running) return nullptr;
return passive;
}
static bool _getPartitionSha256(const esp_partition_t* partition, char* out, size_t outSize) {
if (!partition || !out || outSize < SHA256_HEX_BUFFER_SIZE) return false;
esp_app_desc_t desc;
if (esp_ota_get_partition_description(partition, &desc) != ESP_OK) return false;
sha256BytesToHex(desc.app_elf_sha256, out, outSize);
return true;
}
bool getRunningPartitionSha256(char* out, size_t outSize) {
return _getPartitionSha256(esp_ota_get_running_partition(), out, outSize);
}
bool getOtherPartitionSha256(char* out, size_t outSize) {
return _getPartitionSha256(_getPassiveOtaPartition(), out, outSize);
}
bool getOtherPartitionImageDescriptor(ImageDescriptor::Descriptor& out) {
const esp_partition_t* partition = _getPassiveOtaPartition();
if (!partition) return false;
uint8_t buf[ImageDescriptor::IMAGE_OFFSET + sizeof(ImageDescriptor::Descriptor)];
if (esp_partition_read(partition, 0, buf, sizeof(buf)) != ESP_OK) return false;
return ImageDescriptor::parseFromImageStart(buf, sizeof(buf), out);
}
bool scrubOtaImageHeader(const esp_partition_t* partition) {
if (partition == nullptr) return false;
esp_err_t err = esp_partition_erase_range(partition, 0, OTA_PARTITION_SCRUB_SIZE);
if (err != ESP_OK) {
LOG_ERROR("Failed to scrub rejected OTA image header: %s", esp_err_to_name(err));
return false;
}
LOG_INFO("Scrubbed rejected OTA image header from passive partition");
return true;
}
FirmwareRollbackResult attemptFirmwareRollback(const char* reason) {
// Refuse up front while nothing has been touched. Calling setRestartSystem
// and reacting to its false return is not equivalent: its uptime-gate branch
// LATCHES the request (a ghost restart would fire minutes after we reported
// failure), and its already-scheduled branch would make us restore the boot
// partition of an in-flight rollback that was already reported successful.
if (_restartTaskHandle != NULL || !CrashMonitor::canRestartNow()) {
LOG_WARNING("Rollback: refused, restart already scheduled or blocked");
return FirmwareRollbackResult::RESTART_BLOCKED;
}
const esp_partition_t* running = esp_ota_get_running_partition();
const esp_partition_t* passive = _getPassiveOtaPartition();
if (!passive) {
LOG_ERROR("Rollback: no passive OTA partition found");
return FirmwareRollbackResult::INVALID_IMAGE;
}
// image_validate below only proves the image is well formed, not that it can run here.
// A complete image of the other product can sit in the passive slot (an update refused
// late enough to have been fully written), and booting it fails at PSRAM init, before
// any code that could recover.
ImageDescriptor::Verdict verdict = AppImageDescriptor::validatePartition(passive, false);
if (!ImageDescriptor::accepts(verdict)) {
LOG_ERROR("Rollback: image in %s is not compatible with this device (%s)", passive->label, ImageDescriptor::verdictToString(verdict));
return FirmwareRollbackResult::INVALID_IMAGE;
}
// The real gate: esp_ota_set_boot_partition runs full image_validate on the
// target and refuses an erased/partial/corrupt image with ESP_ERR_OTA_VALIDATE_FAILED.
esp_err_t err = esp_ota_set_boot_partition(passive);
if (err != ESP_OK) {
LOG_ERROR("Rollback: esp_ota_set_boot_partition(%s) failed: %s", passive->label, esp_err_to_name(err));
return FirmwareRollbackResult::INVALID_IMAGE;
}
// Disarm the crash-driven auto-rollback BEFORE scheduling the restart: the
// restart task can preempt this one, and the device must never reboot with
// the flag unset (the crash ladder would bounce straight back to the image
// the operator just left). The pending-OTA record is deliberately NOT
// cleared: it belongs to the job that flashed the currently running image,
// and the post-reboot validation reporting that job FAILED
// (sha256_mismatch_firmware_rollback) is its correct terminal status.
CrashMonitor::markRollbackTried();
if (!setRestartSystem(reason)) {
// Residual race only (both refusal branches were pre-checked above).
// setRestartSystem holds a refused restart rather than cancelling it, so
// undo everything or that held restart would silently execute the
// rollback after we report failure.
CrashMonitor::clearRollbackTried();
esp_err_t restoreErr = esp_ota_set_boot_partition(running);
if (restoreErr != ESP_OK) {
// Practically unreachable (the running image validated at boot); if it
// happens the next reboot boots the passive slot despite the failure report.
LOG_FATAL("Rollback: failed to restore boot partition %s: %s", running->label, esp_err_to_name(restoreErr));
}
LOG_WARNING("Rollback: restart refused, boot partition restored to %s", running->label);
return FirmwareRollbackResult::RESTART_BLOCKED;
}
LOG_WARNING("Rollback: boot partition switched to %s, restarting", passive->label);
return FirmwareRollbackResult::SUCCESS;
}
// Print functions
// -----------------------------
void printDeviceStatusStatic()
{
SystemStaticInfo *info = (SystemStaticInfo*)ps_malloc(sizeof(SystemStaticInfo));
if (!info) {
LOG_ERROR("Failed to allocate SystemStaticInfo in PSRAM");
return;
}
populateSystemStaticInfo(*info);
LOG_DEBUG("--- Static System Info ---");
LOG_DEBUG("Product: %s (%s) | Line: %s | PCB: %s%s", info->fullProductName, info->productName,
info->productLine, info->pcbRevision, info->communityMode ? " | community mode" : "");
LOG_DEBUG("Company: %s | Author: %s", info->companyName, info->author);
LOG_DEBUG("Firmware: %s | Build: %s %s", info->buildVersion, info->buildDate, info->buildTime);
LOG_DEBUG("Sketch MD5: %s | Partition app name: %s", info->sketchMD5, info->partitionAppName);
LOG_DEBUG("Flash: %lu bytes, %lu Hz | PSRAM: %lu bytes", info->flashChipSizeBytes, info->flashChipSpeedHz, info->psramSizeBytes);
LOG_DEBUG("Chip: %s, rev %u, cores %u, id 0x%llx, CPU: %lu MHz", info->chipModel, info->chipRevision, info->chipCores, info->chipId, info->cpuFrequencyMHz);
LOG_DEBUG("SDK: %s | Core: %s", info->sdkVersion, info->coreVersion);
LOG_DEBUG("Device ID: %s", info->deviceId);
LOG_DEBUG("Monitoring: %lu crashes (%lu consecutive), %lu resets (%lu consecutive) | Last reset: %s", info->crashCount, info->consecutiveCrashCount, info->resetCount, info->consecutiveResetCount, info->lastResetReasonString);
free(info);
LOG_DEBUG("------------------------");
}
void printDeviceStatusDynamic()
{
SystemDynamicInfo *info = (SystemDynamicInfo*)ps_malloc(sizeof(SystemDynamicInfo));
if (!info) {
LOG_ERROR("Failed to allocate SystemDynamicInfo in PSRAM");
return;
}
populateSystemDynamicInfo(*info);
LOG_DEBUG("--- Dynamic System Info ---");
char uptimeHuman[DURATION_FORMAT_BUFFER_SIZE];
DurationFormat::humanizeDuration(info->uptimeMilliseconds, uptimeHuman, sizeof(uptimeHuman));
LOG_DEBUG(
"Uptime: %s | Timestamp: %s | Temperature: %.2f C",
uptimeHuman, info->currentTimestampIso, info->temperatureCelsius
);
LOG_DEBUG("Heap: %lu total, %lu free (%.1f%%), %lu used (%.1f%%), %lu min free, %lu max alloc",
info->heapTotalBytes,
info->heapFreeBytes, info->heapFreePercentage,
info->heapUsedBytes, info->heapUsedPercentage,
info->heapMinFreeBytes, info->heapMaxAllocBytes
);
if (info->psramFreeBytes > 0 || info->psramUsedBytes > 0) {
LOG_DEBUG("PSRAM: %lu total, %lu free (%.1f%%), %lu used (%.1f%%), %lu min free, %lu max alloc",
info->psramTotalBytes,
info->psramFreeBytes, info->psramFreePercentage,
info->psramUsedBytes, info->psramUsedPercentage,
info->psramMinFreeBytes, info->psramMaxAllocBytes
);
}
LOG_DEBUG("LittleFS: %lu total, %lu free (%.1f%%), %lu used (%.1f%%)",
info->littlefsTotalBytes,
info->littlefsFreeBytes, info->littlefsFreePercentage,
info->littlefsUsedBytes, info->littlefsUsedPercentage
);
LOG_DEBUG("NVS: %lu total, %lu free (%.1f%%), %lu used (%.1f%%), %u namespaces",
info->totalUsableEntries, info->availableEntries, info->availableEntriesPercentage,
info->usedEntries, info->usedEntriesPercentage, info->namespaceCount
);
if (info->wifiConnected) {
LOG_DEBUG("WiFi: Connected to '%s' (BSSID: %s) | RSSI %ld dBm | MAC %s", info->wifiSsid, info->wifiBssid, info->wifiRssi, info->wifiMacAddress);
LOG_DEBUG("WiFi: IP %s | Gateway %s | DNS %s | Subnet %s", info->wifiLocalIp, info->wifiGatewayIp, info->wifiDnsIp, info->wifiSubnetMask);
} else {
LOG_DEBUG("WiFi: Disconnected | MAC %s", info->wifiMacAddress);
}
if (info->ethEnabled) {
if (info->ethLinkUp) {
LOG_DEBUG("Ethernet: Link up %u Mbps %s duplex | IP %s | MAC %s", (unsigned)info->ethLinkSpeedMbps,
info->ethFullDuplex ? "full" : "half", info->ethLocalIp, info->ethMacAddress);
} else {
LOG_DEBUG("Ethernet: Link down | MAC %s", info->ethMacAddress);
}
}
LOG_DEBUG("Active interface: %s", info->activeInterface);
free(info);
LOG_DEBUG("-------------------------");
}
void updateStatistics() {
// The only statistic which is (currently) updated manually here is the log count
statistics.logVerbose = AdvancedLogger::getVerboseCount();
statistics.logDebug = AdvancedLogger::getDebugCount();
statistics.logInfo = AdvancedLogger::getInfoCount();
statistics.logWarning = AdvancedLogger::getWarningCount();
statistics.logError = AdvancedLogger::getErrorCount();
statistics.logFatal = AdvancedLogger::getFatalCount();
statistics.logDropped = AdvancedLogger::getDroppedCount();
LOG_DEBUG("Statistics updated");
}
void printStatistics() {
updateStatistics();
LOG_DEBUG("--- Statistics ---");
LOG_DEBUG("Statistics - ADE7953: %llu total interrupts | %llu handled interrupts | %llu zx interrupts | %llu zxto interrupts | %llu service passes | %llu unhandled | %llu readings | %llu reading failures",
statistics.ade7953TotalInterrupts,
statistics.ade7953TotalHandledInterrupts,
statistics.ade7953ZxInterrupts,
statistics.ade7953ZxtoInterrupts,
statistics.ade7953ServicePasses,
statistics.ade7953UnhandledInterrupts,
statistics.ade7953ReadingCount,
statistics.ade7953ReadingCountFailure
);
LOG_DEBUG("Statistics - MQTT: %llu messages published | %llu errors | %llu connections | %llu connection errors | %llu meter points dropped | %llu grid points dropped",
statistics.mqttMessagesPublished,
statistics.mqttMessagesPublishedError,
statistics.mqttConnections,
statistics.mqttConnectionErrors,
statistics.mqttMeterPointsDropped,
statistics.mqttGridPointsDropped
);
LOG_DEBUG("Statistics - Custom MQTT: %llu messages published | %llu errors",
statistics.customMqttMessagesPublished,
statistics.customMqttMessagesPublishedError
);
LOG_DEBUG("Statistics - Modbus: %llu requests | %llu errors",
statistics.modbusRequests,
statistics.modbusRequestsError
);
LOG_DEBUG("Statistics - InfluxDB: %llu uploads | %llu errors",
statistics.influxdbUploadCount,
statistics.influxdbUploadCountError
);
LOG_DEBUG("Statistics - WiFi: %llu connections | %llu errors",
statistics.wifiConnection,
statistics.wifiConnectionError
);
LOG_DEBUG("Statistics - Web Server: %llu requests | %llu errors",
statistics.webServerRequests,
statistics.webServerRequestsError
);
LOG_DEBUG("Statistics - Log: %llu verbose | %llu debug | %llu info | %llu warning | %llu error | %llu fatal, %llu dropped",
statistics.logVerbose,
statistics.logDebug,
statistics.logInfo,
statistics.logWarning,
statistics.logError,
statistics.logFatal,
statistics.logDropped
);
LOG_DEBUG("-------------------");
}
void statisticsToJson(Statistics& statistics, JsonDocument &jsonDocument) {
// Update to have latest values
updateStatistics();
// ADE7953 statistics
jsonDocument["ade7953"]["totalInterrupts"] = statistics.ade7953TotalInterrupts;
jsonDocument["ade7953"]["totalHandledInterrupts"] = statistics.ade7953TotalHandledInterrupts;
jsonDocument["ade7953"]["zxInterrupts"] = statistics.ade7953ZxInterrupts;
jsonDocument["ade7953"]["zxtoInterrupts"] = statistics.ade7953ZxtoInterrupts;
jsonDocument["ade7953"]["servicePasses"] = statistics.ade7953ServicePasses;
jsonDocument["ade7953"]["unhandledInterrupts"] = statistics.ade7953UnhandledInterrupts;
jsonDocument["ade7953"]["readingCount"] = statistics.ade7953ReadingCount;
jsonDocument["ade7953"]["readingCountFailure"] = statistics.ade7953ReadingCountFailure;
// MQTT statistics
jsonDocument["mqtt"]["messagesPublished"] = statistics.mqttMessagesPublished;
jsonDocument["mqtt"]["messagesPublishedError"] = statistics.mqttMessagesPublishedError;
jsonDocument["mqtt"]["connections"] = statistics.mqttConnections;
jsonDocument["mqtt"]["connectionErrors"] = statistics.mqttConnectionErrors;
jsonDocument["mqtt"]["meterPointsDropped"] = statistics.mqttMeterPointsDropped;
jsonDocument["mqtt"]["gridPointsDropped"] = statistics.mqttGridPointsDropped;
// Custom MQTT statistics
jsonDocument["customMqtt"]["messagesPublished"] = statistics.customMqttMessagesPublished;
jsonDocument["customMqtt"]["messagesPublishedError"] = statistics.customMqttMessagesPublishedError;
// Modbus statistics
jsonDocument["modbus"]["requests"] = statistics.modbusRequests;
jsonDocument["modbus"]["requestsError"] = statistics.modbusRequestsError;
// InfluxDB statistics
jsonDocument["influxdb"]["uploadCount"] = statistics.influxdbUploadCount;
jsonDocument["influxdb"]["uploadCountError"] = statistics.influxdbUploadCountError;
// WiFi statistics
jsonDocument["wifi"]["connection"] = statistics.wifiConnection;
jsonDocument["wifi"]["connectionError"] = statistics.wifiConnectionError;
// Web Server statistics
jsonDocument["webServer"]["requests"] = statistics.webServerRequests;
jsonDocument["webServer"]["requestsError"] = statistics.webServerRequestsError;
// Log statistics
jsonDocument["log"]["verbose"] = statistics.logVerbose;
jsonDocument["log"]["debug"] = statistics.logDebug;
jsonDocument["log"]["info"] = statistics.logInfo;
jsonDocument["log"]["warning"] = statistics.logWarning;
jsonDocument["log"]["error"] = statistics.logError;
jsonDocument["log"]["fatal"] = statistics.logFatal;
jsonDocument["log"]["dropped"] = statistics.logDropped;
LOG_VERBOSE("Statistics converted to JSON");
}
// Helper functions
// -----------------------------
static void _factoryReset() { // No logger here it is likely destroyed already
Serial.println("[WARNING] Factory reset requested");
Led::blinkRedFast(Led::PRIO_CRITICAL); // The critical layer carries its own brightness floor
clearAllPreferences();
// clearAllPreferences() shares its exclusion list with backup/restore, where the web
// password must never travel. A factory reset is the opposite case: it has to hand the
// device over on the default password, so wipe auth here. The server re-seeds the
// default on the next boot when the namespace is empty.
Preferences authPreferences;
if (authPreferences.begin(PREFERENCES_NAMESPACE_AUTH, false)) {
authPreferences.clear();
authPreferences.end();
} else {
Serial.println("[ERROR] Failed to clear the web password");
}
// clearAllPreferences() deliberately skips every "nvs.*" namespace (see its own
// comment), which includes the WiFi driver's own persisted association
// (nvs.net80211 - the same store CustomWifi::_hasStoredCredentials() reads and
// resetWifi() erases via this same call). Without this, a factory reset left the
// device rejoining its old network on the next boot instead of coming up unprovisioned.
esp_err_t err = esp_wifi_restore();
if (err != ESP_OK) {
Serial.printf("[ERROR] Failed to erase stored WiFi credentials: %s\n", esp_err_to_name(err));
}
// Unmount first: formatting a still-mounted filesystem is unsupported by esp_littlefs
// and every other module (crash archive, log file, waveform captures) still holds it
// open at this point since none of them are in the stop-services list above.
LittleFS.end();
// RTC memory is neither NVS nor LittleFS, so nothing above touches it: without
// this the reset device inherits the old crash/reset counts and can boot
// straight into safe mode.
CrashMonitor::clearPersistentState();
Serial.println("[WARNING] Formatting LittleFS. This will take some time.");
LittleFS.format();
// Removed ESP.restart() call since the factory reset must be called only from the restart task
}
bool isFirstBootDone() {
Preferences preferences;
if (!preferences.begin(PREFERENCES_NAMESPACE_GENERAL, true)) {
LOG_DEBUG("Could not open preferences namespace: %s. Assuming first boot", PREFERENCES_NAMESPACE_GENERAL);
return false;
}
bool firstBoot = preferences.getBool(IS_FIRST_BOOT_DONE_KEY, false);
preferences.end();
return firstBoot;
}
void setFirstBootDone() { // No arguments because the only way to set first boot done to false it through a complete wipe - thus automatically setting it to "false"
Preferences preferences;
if (!preferences.begin(PREFERENCES_NAMESPACE_GENERAL, false)) {
LOG_ERROR("Failed to open preferences namespace: %s", PREFERENCES_NAMESPACE_GENERAL);
return;
}
preferences.putBool(IS_FIRST_BOOT_DONE_KEY, true);
preferences.end();
}
void createAllNamespaces() {
// Initialize all known app namespaces on first boot
const char* namespacesToCreate[] = {TO_INITIALIZE_NVS_NAMESPACES_LIST};
Preferences preferences;
for (const char* ns : namespacesToCreate) {
preferences.begin(ns, false);
preferences.end();
}
LOG_DEBUG("All namespaces created");
}
void clearAllPreferences() {
// Clear all app namespaces during factory reset, skipping only critical ones
// Critical namespaces: factory-provisioned data (certs, calibration) and device-specific data
const char* excludedNamespaces[] = {EXCLUDED_NVS_NAMESPACES_LIST};
nvs_iterator_t it = nullptr;
esp_err_t err = nvs_entry_find("nvs", nullptr, NVS_TYPE_ANY, &it);
if (err != ESP_OK) {
LOG_WARNING("Could not initialize NVS iterator for clearing preferences (error %d)", err);
return;
}
Preferences preferences;
uint32_t clearedCount = 0;
uint32_t loops = 0;
while (err == ESP_OK && loops < MAX_LOOP_ITERATIONS) {
loops++;
nvs_entry_info_t info;
nvs_entry_info(it, &info);
// Skip system namespaces (handled separately by ESP)
// Most likely they would be already handled by excludedNamespaces, but better safe than sorry
if (strncmp(info.namespace_name, "nvs.", 4) == 0) {
err = nvs_entry_next(&it);
continue;
}
// Check if this namespace should be skipped (critical data)
bool isExcluded = false;
for (const char* excluded : excludedNamespaces) {
if (strcmp(info.namespace_name, excluded) == 0) {
isExcluded = true;
break;
}
}
// Clear non-critical namespaces
if (!isExcluded && preferences.begin(info.namespace_name, false)) {
preferences.clear();
preferences.end();
clearedCount++;
}
err = nvs_entry_next(&it);
if (err != ESP_OK && err != ESP_ERR_NVS_NOT_FOUND) {
LOG_ERROR("Could not advance NVS iterator (error %d)", err);
break;
}
}
if (it != nullptr) {
nvs_release_iterator(it);
}
LOG_WARNING("Cleared %ld preferences (factory/device-specific data preserved)", clearedCount);
}
bool probeTcp(const char* host, uint16_t port, uint32_t timeoutMs) {
WiFiClient client;
client.setTimeout(timeoutMs);
if (!client.connect(host, port)) return false;
client.stop();
return true;
}
void getDeviceId(char* deviceId, size_t maxLength) {
uint8_t mac[6];
esp_efuse_mac_get_default(mac);
// Use lowercase hex formatting without colons
snprintf(deviceId, maxLength, "%02x%02x%02x%02x%02x%02x",
mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
}
void generateHexToken(char* out, size_t outSize) {
if (!out || outSize < 17) return;
snprintf(out, outSize, "%08x%08x", esp_random(), esp_random());
}
uint64_t calculateExponentialBackoff(uint64_t attempt, uint64_t initialInterval, uint64_t maxInterval, uint64_t multiplier) {
return BackoffSchedule::delayForAttempt(attempt, initialInterval, maxInterval, multiplier);
}
// === LittleFS FILE OPERATIONS ===
bool listLittleFsFiles(JsonDocument &doc, const char* folderPath) {
if (folderPath && strlen(folderPath) > 0) {
// Ensure folder path starts with /
char normalizedPath[NAME_BUFFER_SIZE];
if (folderPath[0] != '/') {
snprintf(normalizedPath, sizeof(normalizedPath), "/%s", folderPath);
} else {
snprintf(normalizedPath, sizeof(normalizedPath), "%s", folderPath);
}
// Check if the folder exists
if (!LittleFS.exists(normalizedPath)) {
LOG_DEBUG("Folder does not exist: %s", normalizedPath);
return true; // Return true with empty doc - not an error
}
return _listLittleFsFilesRecursive(doc, normalizedPath, normalizedPath, 0);
}
return _listLittleFsFilesRecursive(doc, "/", nullptr, 0);
}
static bool _listLittleFsFilesRecursive(JsonDocument &doc, const char* dirname, const char* basePath, uint8_t levels) {
File root = LittleFS.open(dirname);
if (!root) {
LOG_ERROR("Failed to open LittleFS directory: %s", dirname);
return false;
}
if (!root.isDirectory()) {
LOG_ERROR("Path is not a directory: %s", dirname);
root.close();
return false;
}
File file = root.openNextFile();
uint32_t loops = 0;
while (file && loops < MAX_LOOP_ITERATIONS) {
loops++;
const char* filepath = file.path();
if (file.isDirectory()) {
// Recursively list subdirectory contents (limit depth to prevent infinite recursion)
if (levels < 5) {
_listLittleFsFilesRecursive(doc, filepath, basePath, levels + 1);
}
} else {
const char* displayPath = filepath;
// If we have a basePath, make paths relative to it
if (basePath) {
size_t baseLen = strlen(basePath);
if (strncmp(filepath, basePath, baseLen) == 0) {
displayPath = filepath + baseLen;
// Skip leading slash after base path
if (displayPath[0] == '/') displayPath++;
}
} else {
// Remove leading slash for consistency (global listing)
if (filepath[0] == '/') displayPath++;
}
// Add file with its size to the JSON document
doc[displayPath] = file.size();
}
file = root.openNextFile();
}
root.close();
return true;
}
bool getLittleFsFileContent(const char* filepath, char* buffer, size_t bufferSize) {
if (!filepath || !buffer || bufferSize == 0) {
LOG_ERROR("Invalid arguments provided");
return false;
}
// Check if file exists
if (!LittleFS.exists(filepath)) {
LOG_DEBUG("File not found: %s", filepath);
return false;
}
File file = LittleFS.open(filepath, FILE_READ);
if (!file) {
LOG_ERROR("Failed to open file: %s", filepath);
return false;
}
size_t bytesRead = file.readBytes(buffer, bufferSize - 1);
buffer[bytesRead] = '\0'; // Null-terminate the string
file.close();
LOG_DEBUG("Successfully read file: %s (%d bytes)", filepath, bytesRead);
return true;
}
const char* getContentTypeFromFilename(const char* filename) {
if (!filename) return "application/octet-stream";
// Find the file extension
const char* ext = strrchr(filename, '.');
if (!ext) return "application/octet-stream";
// Convert to lowercase for comparison
char extension[16];
size_t extLen = strlen(ext);
if (extLen >= sizeof(extension)) return "application/octet-stream";
for (size_t i = 0; i < extLen; i++) {
extension[i] = (char)tolower(ext[i]);
}
extension[extLen] = '\0';
// Common file types used in the project
if (strcmp(extension, ".json") == 0) return "application/json";
if (strcmp(extension, ".txt") == 0) return "text/plain";
if (strcmp(extension, ".log") == 0) return "text/plain";
if (strcmp(extension, ".csv") == 0) return "text/csv";
if (strcmp(extension, ".xml") == 0) return "application/xml";
if (strcmp(extension, ".html") == 0) return "text/html";
if (strcmp(extension, ".css") == 0) return "text/css";
if (strcmp(extension, ".js") == 0) return "application/javascript";
if (strcmp(extension, ".bin") == 0) return "application/octet-stream";
if (strcmp(extension, ".gz") == 0) return "application/gzip";
if (strcmp(extension, ".tar") == 0) return "application/x-tar";
return "application/octet-stream";
}
bool compressFile(const char* filepath) {
if (!filepath) {
LOG_ERROR("Invalid file path");
return false;
}
char sourcePath[NAME_BUFFER_SIZE];
char destinationPath[NAME_BUFFER_SIZE + 3]; // Plus .gz
char tempPath[NAME_BUFFER_SIZE + 7]; // Plus .gz.tmp
snprintf(sourcePath, sizeof(sourcePath), "%s", filepath);
snprintf(destinationPath, sizeof(destinationPath), "%s.gz", sourcePath);
snprintf(tempPath, sizeof(tempPath), "%s.gz.tmp", sourcePath);
if (!LittleFS.exists(sourcePath)) {
LOG_WARNING("No finished csv to compress: %s", sourcePath);
return false;
}
// Remove any existing .gz.tmp file before starting
if (LittleFS.exists(tempPath)) {
LOG_DEBUG("Found existing temp file %s. Removing it", tempPath);
if (!LittleFS.remove(tempPath)) {
LOG_ERROR("Failed to remove existing temp file: %s", tempPath);
return false;
}
}
// Remove any existing .gz file before renaming (atomic replace)
if (LittleFS.exists(destinationPath)) {
LOG_DEBUG("Found existing compressed file %s. Removing it", destinationPath);
if (!LittleFS.remove(destinationPath)) {
LOG_ERROR("Failed to remove existing compressed file: %s", destinationPath);
return false;
}
}
File srcFile = LittleFS.open(sourcePath, FILE_READ);
if (!srcFile) {
LOG_ERROR("Failed to open source file: %s", sourcePath);
return false;
}
size_t sourceSize = srcFile.size();
File tempFile = LittleFS.open(tempPath, FILE_WRITE);
if (!tempFile) {
LOG_ERROR("Failed to open temporary file: %s", tempPath);
srcFile.close();
return false;
}
size_t compressedSize = LZPacker::compress(&srcFile, sourceSize, &tempFile);
srcFile.close();
tempFile.close();
if (compressedSize > 0) {
LOG_DEBUG("Compressed finished CSV %s (%zu bytes) -> %s (%zu bytes)", sourcePath, sourceSize, tempPath, compressedSize);
// Rename temp file to final .gz name
if (!LittleFS.rename(tempPath, destinationPath)) {
LOG_ERROR("Failed to rename temporary file %s to final %s", tempPath, destinationPath);
// Clean up temp file
LittleFS.remove(tempPath);
return false;
}
if (!LittleFS.remove(sourcePath)) {
LOG_WARNING("Could not delete original %s after compression", sourcePath);
return false; // Compression succeeded, but cleanup failed - treat as failure
}
} else {
LOG_ERROR("Failed to compress finished CSV %s", sourcePath);
// Clean up temp file if created
LittleFS.remove(tempPath);
return false;
}
LOG_DEBUG("Successfully compressed %s (%zu bytes) to %s (%zu bytes)", sourcePath, sourceSize, destinationPath, compressedSize);
return true;
}
void migrateCsvToGzip(const char* dirPath, const char* excludePrefix) {
LOG_DEBUG("Starting CSV -> gzip migration in %s", dirPath);
if (!LittleFS.exists(dirPath)) {
LOG_DEBUG("Energy folder not present, nothing to migrate");
return;
}
File dir = LittleFS.open(dirPath);
if (!dir) {
LOG_WARNING("Cannot open dir %s", dirPath);
return;
}
dir.rewindDirectory();
File file = dir.openNextFile();
uint32_t loops = 0;
while (file && loops < MAX_LOOP_ITERATIONS) {
loops++;
if (!file.isDirectory()) {
const char* path = file.name();
char fullPath[NAME_BUFFER_SIZE];
snprintf(fullPath, sizeof(fullPath), "%s/%s", dirPath, path);
if (excludePrefix && startsWith(fullPath, excludePrefix)) {
LOG_DEBUG("Skipping file %s due to exclude prefix", fullPath);
file.close(); // Close file handle before continuing
file = dir.openNextFile();
continue;
}
if (endsWith(fullPath, ".csv")) {
file.close(); // Close file handle before attempting compression/deletion
LOG_DEBUG("Migrating %s -> %s.gz", fullPath, fullPath);
if (compressFile(fullPath)) {
LOG_INFO("Compressed and removed original %s", fullPath);
} else {
LOG_ERROR("Compression failed for %s", fullPath);
}
} else {
file.close(); // Close file handle if not processing
}
} else {
file.close(); // Close directory handle
}
file = dir.openNextFile();
}
dir.close();
LOG_DEBUG("CSV -> gzip migration finished");
}
// === ENERGY FILE CONSOLIDATION ===
static bool _ensureDirectoryExists(const char* dirPath) {
if (LittleFS.exists(dirPath)) return true;
if (LittleFS.mkdir(dirPath)) {
LOG_DEBUG("Created directory: %s", dirPath);
return true;
}
LOG_ERROR("Failed to create directory: %s", dirPath);
return false;
}
static bool _decompressGzipFile(const char* gzPath, const char* outputPath) {
if (!LittleFS.exists(gzPath)) {
LOG_WARNING("Gzip file not found: %s", gzPath);
return false;
}
GzUnpacker *unpacker = new GzUnpacker();
if (!unpacker) {
LOG_ERROR("Failed to allocate GzUnpacker");
return false;
}
unpacker->haltOnError(false);
bool success = unpacker->gzExpander(LittleFS, gzPath, LittleFS, outputPath);
if (!success) {
LOG_ERROR("Failed to decompress %s (error %d)", gzPath, unpacker->tarGzGetError());
}
delete unpacker;
return success;
}
static bool _appendFileToFile(const char* srcPath, File &destFile, bool skipHeader) {
File srcFile = LittleFS.open(srcPath, FILE_READ);
if (!srcFile) {
LOG_ERROR("Failed to open source file: %s", srcPath);
return false;
}
// Skip header line if requested
if (skipHeader && srcFile.available()) {
// Read and discard the first line (header)
uint32_t loops = 0;
while (srcFile.available() && loops < MAX_LOOP_ITERATIONS) {
loops++;
int c = srcFile.read();
if (c == '\n') break;
}
}
// Copy remaining content
uint8_t buffer[512];
uint32_t loops = 0;
while (srcFile.available() && loops < MAX_LOOP_ITERATIONS) {
loops++;
size_t bytesRead = srcFile.read(buffer, sizeof(buffer));
if (bytesRead > 0) {
destFile.write(buffer, bytesRead);
}
}
srcFile.close();
return true;
}
bool migrateEnergyFilesToDailyFolder() {
LOG_DEBUG("Starting energy files migration to daily folder");
// Ensure base energy directory exists
if (!_ensureDirectoryExists(ENERGY_CSV_PREFIX)) return false;
// Check if there are any files in the root energy folder to migrate
File dir = LittleFS.open(ENERGY_CSV_PREFIX);
if (!dir) {
LOG_DEBUG("Energy folder not present, nothing to migrate");
return true;
}
// Ensure daily subdirectory exists
if (!_ensureDirectoryExists(ENERGY_CSV_DAILY_PREFIX)) {
dir.close();
return false;
}
dir.rewindDirectory();
uint32_t migratedCount = 0;
uint32_t loops = 0;
File file = dir.openNextFile();
while (file && loops < MAX_LOOP_ITERATIONS) {
loops++;
if (!file.isDirectory()) {
const char* filename = file.name();
// Only migrate .csv.gz files (daily energy files)
if (endsWith(filename, ".csv.gz")) {
char srcPath[NAME_BUFFER_SIZE];
char destPath[NAME_BUFFER_SIZE];
snprintf(srcPath, sizeof(srcPath), "%s/%s", ENERGY_CSV_PREFIX, filename);
snprintf(destPath, sizeof(destPath), "%s/%s", ENERGY_CSV_DAILY_PREFIX, filename);
file.close(); // Close before rename
if (LittleFS.rename(srcPath, destPath)) {
LOG_DEBUG("Migrated %s -> %s", srcPath, destPath);
migratedCount++;
} else {
LOG_ERROR("Failed to migrate %s", srcPath);
}
file = dir.openNextFile();
continue;
}
// Also migrate uncompressed .csv files (except today's)
else if (endsWith(filename, ".csv")) {
char srcPath[NAME_BUFFER_SIZE];
char destPath[NAME_BUFFER_SIZE];
snprintf(srcPath, sizeof(srcPath), "%s/%s", ENERGY_CSV_PREFIX, filename);
snprintf(destPath, sizeof(destPath), "%s/%s", ENERGY_CSV_DAILY_PREFIX, filename);
file.close(); // Close before rename
if (LittleFS.rename(srcPath, destPath)) {
LOG_DEBUG("Migrated %s -> %s", srcPath, destPath);
migratedCount++;
} else {
LOG_ERROR("Failed to migrate %s", srcPath);
}
file = dir.openNextFile();
continue;
}
}
file.close();
file = dir.openNextFile();
}
dir.close();
if (migratedCount > 0) {
LOG_INFO("Migrated %lu energy files to daily folder", migratedCount);
} else {
LOG_DEBUG("No energy files needed migration");
}
return true;
}
bool consolidateDailyFilesToMonthly(const char* yearMonth, const char* excludeDate) {
if (!yearMonth || strlen(yearMonth) != 7) { // Format: YYYY-MM
LOG_ERROR("Invalid yearMonth format: %s (expected YYYY-MM)", yearMonth ? yearMonth : "null");
return false;
}
LOG_DEBUG("Starting daily -> monthly consolidation for %s (excluding: %s)", yearMonth, excludeDate ? excludeDate : "none");
// Ensure monthly directory exists
if (!_ensureDirectoryExists(ENERGY_CSV_MONTHLY_PREFIX)) return false;
// Check if daily folder exists
if (!LittleFS.exists(ENERGY_CSV_DAILY_PREFIX)) {
LOG_DEBUG("Daily folder does not exist, nothing to consolidate");
return true;
}
// Prepare paths
char monthlyTempPath[NAME_BUFFER_SIZE];
char monthlyFinalPath[NAME_BUFFER_SIZE];
char monthlyGzPath[NAME_BUFFER_SIZE];
char monthlyGzTempPath[NAME_BUFFER_SIZE];
snprintf(monthlyTempPath, sizeof(monthlyTempPath), "%s/%s.csv.tmp", ENERGY_CSV_MONTHLY_PREFIX, yearMonth);
snprintf(monthlyFinalPath, sizeof(monthlyFinalPath), "%s/%s.csv", ENERGY_CSV_MONTHLY_PREFIX, yearMonth);
snprintf(monthlyGzPath, sizeof(monthlyGzPath), "%s/%s.csv.gz", ENERGY_CSV_MONTHLY_PREFIX, yearMonth);
snprintf(monthlyGzTempPath, sizeof(monthlyGzTempPath), "%s/%s.csv.gz.tmp", ENERGY_CSV_MONTHLY_PREFIX, yearMonth);
// Check if monthly archive already exists - we'll need to decompress and append
bool existingArchive = LittleFS.exists(monthlyGzPath);
if (existingArchive) {
LOG_DEBUG("Monthly archive already exists for %s, will append new daily files", yearMonth);
}
// Clean up any existing temp files from previous failed attempts
if (LittleFS.exists(monthlyTempPath)) LittleFS.remove(monthlyTempPath);
if (LittleFS.exists(monthlyGzTempPath)) LittleFS.remove(monthlyGzTempPath);
// Collect matching daily files (excluding the specified date if provided)
std::vector<char*> dailyFiles;
File dir = LittleFS.open(ENERGY_CSV_DAILY_PREFIX);
if (!dir) {
LOG_ERROR("Failed to open daily folder");
return false;
}
dir.rewindDirectory();
uint32_t loops = 0;
File file = dir.openNextFile();
while (file && loops < MAX_LOOP_ITERATIONS) {
loops++;
if (!file.isDirectory()) {
const char* filename = file.name();
// Check if file matches the month pattern (YYYY-MM-DD.csv.gz)
if (endsWith(filename, ".csv.gz") && strncmp(filename, yearMonth, 7) == 0) {
// Check if this date should be excluded (e.g., today's date)
if (excludeDate && strncmp(filename, excludeDate, 10) == 0) {
LOG_DEBUG("Skipping excluded date: %s", filename);
file.close();
file = dir.openNextFile();
continue;
}
char* fileCopy = (char*)ps_malloc(strlen(filename) + 1);
if (fileCopy) {
strcpy(fileCopy, filename);
dailyFiles.push_back(fileCopy);
} else {
LOG_WARNING("Failed to allocate %zu bytes from PSRAM for filename: %s (free PSRAM: %lu)",
strlen(filename) + 1, filename, ESP.getFreePsram());
}
}
}
file.close();
file = dir.openNextFile();
}
dir.close();
if (dailyFiles.empty()) {
LOG_DEBUG("No daily files found for %s", yearMonth);
return true;
}
LOG_DEBUG("Found %d daily files for %s", dailyFiles.size(), yearMonth);
// Sort files by name (chronological order)
std::sort(dailyFiles.begin(), dailyFiles.end(), [](const char* a, const char* b) {
return strcmp(a, b) < 0;
});
// Create temporary consolidated CSV file
File tempFile = LittleFS.open(monthlyTempPath, FILE_WRITE);
if (!tempFile) {
LOG_ERROR("Failed to create temp file: %s", monthlyTempPath);
for (auto f : dailyFiles) free(f);
return false;
}
bool success = true;
// If existing archive exists, decompress it and copy content first (skip writing new header)
if (existingArchive) {
char existingCsvPath[NAME_BUFFER_SIZE];
snprintf(existingCsvPath, sizeof(existingCsvPath), "%s/_existing_monthly.csv", ENERGY_CSV_MONTHLY_PREFIX);
// Clean up any existing temp file
if (LittleFS.exists(existingCsvPath)) LittleFS.remove(existingCsvPath);
// Decompress existing archive
if (_decompressGzipFile(monthlyGzPath, existingCsvPath)) {
// Append existing content (including header since it's the first content)
if (!_appendFileToFile(existingCsvPath, tempFile, false)) {
LOG_ERROR("Failed to append existing monthly archive content");
success = false;
}
LittleFS.remove(existingCsvPath);
} else {
LOG_ERROR("Failed to decompress existing monthly archive");
success = false;
}
if (!success) {
tempFile.close();
LittleFS.remove(monthlyTempPath);
for (auto f : dailyFiles) free(f);
return false;
}
} else {
// Write header once (only for new archives)
tempFile.println(DAILY_ENERGY_CSV_HEADER);
}
std::vector<char*> processedFiles;
for (size_t i = 0; i < dailyFiles.size() && success; i++) {
char gzPath[NAME_BUFFER_SIZE];
char csvPath[NAME_BUFFER_SIZE];
snprintf(gzPath, sizeof(gzPath), "%s/%s", ENERGY_CSV_DAILY_PREFIX, dailyFiles[i]);
// Create temp path for decompressed CSV
char tempCsvName[NAME_BUFFER_SIZE];
snprintf(tempCsvName, sizeof(tempCsvName), "%s/%sdecomp.csv", ENERGY_CSV_DAILY_PREFIX, TEMPORARY_FILE_PREFIX);
snprintf(csvPath, sizeof(csvPath), "%s", tempCsvName);
// Clean up any existing temp CSV
if (LittleFS.exists(csvPath)) LittleFS.remove(csvPath);
// Decompress the daily file
if (!_decompressGzipFile(gzPath, csvPath)) {
LOG_ERROR("Failed to decompress %s", gzPath);
success = false;
break;
}
// Append to consolidated file (skip header)
if (!_appendFileToFile(csvPath, tempFile, true)) {
LOG_ERROR("Failed to append %s to consolidated file", dailyFiles[i]);
LittleFS.remove(csvPath);
success = false;
break;
}
// Remove temp decompressed file
LittleFS.remove(csvPath);
processedFiles.push_back(dailyFiles[i]);
LOG_VERBOSE("Consolidated %s", dailyFiles[i]);
}
tempFile.close();
if (!success) {
LittleFS.remove(monthlyTempPath);
for (auto f : dailyFiles) free(f);
return false;
}
// Verify temp file has content
File verifyFile = LittleFS.open(monthlyTempPath, FILE_READ);
if (!verifyFile || verifyFile.size() < ENERGY_CONSOLIDATION_MIN_SIZE) {
LOG_ERROR("Consolidated temp file too small or invalid");
if (verifyFile) verifyFile.close();
LittleFS.remove(monthlyTempPath);
for (auto f : dailyFiles) free(f);
return false;
}
verifyFile.close();
// Rename temp to final CSV
if (!LittleFS.rename(monthlyTempPath, monthlyFinalPath)) {
LOG_ERROR("Failed to rename temp file to final CSV");
LittleFS.remove(monthlyTempPath);
for (auto f : dailyFiles) free(f);
return false;
}
// Compress the consolidated CSV
if (!compressFile(monthlyFinalPath)) {
LOG_ERROR("Failed to compress consolidated file");
for (auto f : dailyFiles) free(f);
return false;
}
// Verify compressed file exists and has content
File verifyGz = LittleFS.open(monthlyGzPath, FILE_READ);
if (!verifyGz || verifyGz.size() < ENERGY_CONSOLIDATION_MIN_SIZE) {
LOG_ERROR("Compressed file too small or invalid");
if (verifyGz) verifyGz.close();
for (auto f : dailyFiles) free(f);
return false;
}
size_t finalSize = verifyGz.size();
verifyGz.close();
// Delete original daily files
uint32_t deletedCount = 0;
for (auto filename : processedFiles) {
char gzPath[NAME_BUFFER_SIZE];
snprintf(gzPath, sizeof(gzPath), "%s/%s", ENERGY_CSV_DAILY_PREFIX, filename);
if (LittleFS.remove(gzPath)) {
deletedCount++;
} else {
LOG_WARNING("Failed to delete %s after consolidation", gzPath);
}
}
// Cleanup
for (auto f : dailyFiles) free(f);
LOG_INFO("Consolidated %d daily files for %s into %zu bytes (%lu deleted)",
processedFiles.size(), yearMonth, finalSize, deletedCount);
return true;
}
bool consolidateMonthlyFilesToYearly(const char* year, const char* excludeMonth) {
if (!year || strlen(year) != 4) { // Format: YYYY
LOG_ERROR("Invalid year format: %s (expected YYYY)", year ? year : "null");
return false;
}
LOG_DEBUG("Starting monthly -> yearly consolidation for %s (excluding: %s)", year, excludeMonth ? excludeMonth : "none");
// Ensure yearly directory exists
if (!_ensureDirectoryExists(ENERGY_CSV_YEARLY_PREFIX)) return false;
// Check if monthly folder exists
if (!LittleFS.exists(ENERGY_CSV_MONTHLY_PREFIX)) {
LOG_DEBUG("Monthly folder does not exist, nothing to consolidate");
return true;
}
// Prepare paths
char yearlyTempPath[NAME_BUFFER_SIZE];
char yearlyFinalPath[NAME_BUFFER_SIZE];
char yearlyGzPath[NAME_BUFFER_SIZE];
snprintf(yearlyTempPath, sizeof(yearlyTempPath), "%s/%s.csv.tmp", ENERGY_CSV_YEARLY_PREFIX, year);
snprintf(yearlyFinalPath, sizeof(yearlyFinalPath), "%s/%s.csv", ENERGY_CSV_YEARLY_PREFIX, year);
snprintf(yearlyGzPath, sizeof(yearlyGzPath), "%s/%s.csv.gz", ENERGY_CSV_YEARLY_PREFIX, year);
// Check if yearly archive already exists - we'll need to decompress and append
bool existingArchive = LittleFS.exists(yearlyGzPath);
if (existingArchive) {
LOG_DEBUG("Yearly archive already exists for %s, will append new monthly files", year);
}
// Clean up any existing temp files
if (LittleFS.exists(yearlyTempPath)) LittleFS.remove(yearlyTempPath);
// Collect matching monthly files (excluding the specified month if provided)
std::vector<char*> monthlyFiles;
File dir = LittleFS.open(ENERGY_CSV_MONTHLY_PREFIX);
if (!dir) {
LOG_ERROR("Failed to open monthly folder");
return false;
}
dir.rewindDirectory();
uint32_t loops = 0;
File file = dir.openNextFile();
while (file && loops < MAX_LOOP_ITERATIONS) {
loops++;
if (!file.isDirectory()) {
const char* filename = file.name();
// Check if file matches the year pattern (YYYY-MM.csv.gz)
if (endsWith(filename, ".csv.gz") && strncmp(filename, year, 4) == 0) {
// Check if this month should be excluded (e.g., current month)
if (excludeMonth && strncmp(filename, excludeMonth, 7) == 0) {
LOG_DEBUG("Skipping excluded month: %s", filename);
file.close();
file = dir.openNextFile();
continue;
}
char* fileCopy = (char*)ps_malloc(strlen(filename) + 1);
if (fileCopy) {
strcpy(fileCopy, filename);
monthlyFiles.push_back(fileCopy);
} else {
LOG_WARNING("Failed to allocate %zu bytes from PSRAM for filename: %s (free PSRAM: %lu)",
strlen(filename) + 1, filename, ESP.getFreePsram());
}
}
}
file.close();
file = dir.openNextFile();
}
dir.close();
if (monthlyFiles.empty()) {
LOG_DEBUG("No monthly files found for %s", year);
return true;
}
LOG_DEBUG("Found %d monthly files for %s", monthlyFiles.size(), year);
// Sort files by name (chronological order)
std::sort(monthlyFiles.begin(), monthlyFiles.end(), [](const char* a, const char* b) {
return strcmp(a, b) < 0;
});
// If existing archive, decompress it first as a starting point
if (existingArchive) {
if (!_decompressGzipFile(yearlyGzPath, yearlyTempPath)) {
LOG_ERROR("Failed to decompress existing yearly archive %s", yearlyGzPath);
for (auto f : monthlyFiles) free(f);
return false;
}
LOG_DEBUG("Decompressed existing yearly archive for appending");
}
// Create/append to temporary consolidated CSV file
File tempFile = LittleFS.open(yearlyTempPath, existingArchive ? FILE_APPEND : FILE_WRITE);
if (!tempFile) {
LOG_ERROR("Failed to open temp file: %s", yearlyTempPath);
for (auto f : monthlyFiles) free(f);
return false;
}
// Write header only if new file
if (!existingArchive) {
tempFile.println(DAILY_ENERGY_CSV_HEADER);
}
bool success = true;
std::vector<char*> processedFiles;
for (size_t i = 0; i < monthlyFiles.size() && success; i++) {
char gzPath[NAME_BUFFER_SIZE];
char csvPath[NAME_BUFFER_SIZE];
snprintf(gzPath, sizeof(gzPath), "%s/%s", ENERGY_CSV_MONTHLY_PREFIX, monthlyFiles[i]);
// Create temp path for decompressed CSV
char tempCsvName[NAME_BUFFER_SIZE];
snprintf(tempCsvName, sizeof(tempCsvName), "%s/%sdecomp.csv", ENERGY_CSV_MONTHLY_PREFIX, TEMPORARY_FILE_PREFIX);
snprintf(csvPath, sizeof(csvPath), "%s", tempCsvName);
// Clean up any existing temp CSV
if (LittleFS.exists(csvPath)) LittleFS.remove(csvPath);
// Decompress the monthly file
if (!_decompressGzipFile(gzPath, csvPath)) {
LOG_ERROR("Failed to decompress %s", gzPath);
success = false;
break;
}
// Append to consolidated file (skip header)
if (!_appendFileToFile(csvPath, tempFile, true)) {
LOG_ERROR("Failed to append %s to consolidated file", monthlyFiles[i]);
LittleFS.remove(csvPath);
success = false;
break;
}
// Remove temp decompressed file
LittleFS.remove(csvPath);
processedFiles.push_back(monthlyFiles[i]);
LOG_VERBOSE("Consolidated %s", monthlyFiles[i]);
}
tempFile.close();
if (!success) {
LittleFS.remove(yearlyTempPath);
for (auto f : monthlyFiles) free(f);
return false;
}
// Verify temp file has content
File verifyFile = LittleFS.open(yearlyTempPath, FILE_READ);
if (!verifyFile || verifyFile.size() < ENERGY_CONSOLIDATION_MIN_SIZE) {
LOG_ERROR("Consolidated temp file too small or invalid");
if (verifyFile) verifyFile.close();
LittleFS.remove(yearlyTempPath);
for (auto f : monthlyFiles) free(f);
return false;
}
verifyFile.close();
// Rename temp to final CSV
if (!LittleFS.rename(yearlyTempPath, yearlyFinalPath)) {
LOG_ERROR("Failed to rename temp file to final CSV");
LittleFS.remove(yearlyTempPath);
for (auto f : monthlyFiles) free(f);
return false;
}
// Compress the consolidated CSV
if (!compressFile(yearlyFinalPath)) {
LOG_ERROR("Failed to compress consolidated file");
for (auto f : monthlyFiles) free(f);
return false;
}
// Verify compressed file exists and has content
File verifyGz = LittleFS.open(yearlyGzPath, FILE_READ);
if (!verifyGz || verifyGz.size() < ENERGY_CONSOLIDATION_MIN_SIZE) {
LOG_ERROR("Compressed file too small or invalid");
if (verifyGz) verifyGz.close();
for (auto f : monthlyFiles) free(f);
return false;
}
size_t finalSize = verifyGz.size();
verifyGz.close();
// Delete original monthly files
uint32_t deletedCount = 0;
for (auto filename : processedFiles) {
char gzPath[NAME_BUFFER_SIZE];
snprintf(gzPath, sizeof(gzPath), "%s/%s", ENERGY_CSV_MONTHLY_PREFIX, filename);
if (LittleFS.remove(gzPath)) {
deletedCount++;
} else {
LOG_WARNING("Failed to delete %s after consolidation", gzPath);
}
}
// Cleanup
for (auto f : monthlyFiles) free(f);
LOG_INFO("Consolidated %d monthly files for %s into %zu bytes (%lu deleted)",
processedFiles.size(), year, finalSize, deletedCount);
return true;
}
bool nvsDataToJson(JsonObject &doc) {
LOG_DEBUG("Exporting NVS data to JSON document");
// Add metadata
doc["version"] = 1;
doc["type"] = "configuration";
char deviceId[DEVICE_ID_BUFFER_SIZE];
getDeviceId(deviceId, sizeof(deviceId));
doc["deviceId"] = deviceId;
doc["firmwareVersion"] = FIRMWARE_BUILD_VERSION;
doc["sketchMD5"] = ESP.getSketchMD5().c_str();
char timestamp[TIMESTAMP_ISO_BUFFER_SIZE];
CustomTime::getTimestampIso(timestamp, sizeof(timestamp));
doc["timestamp"] = timestamp;
// Create nvs object
doc["nvs"].to<JsonObject>();
// Namespaces to exclude from backup (sensitive, device-specific, or auto-generated data)
const char* excludedNamespaces[] = {EXCLUDED_NVS_NAMESPACES_LIST};
const size_t excludedCount = sizeof(excludedNamespaces) / sizeof(excludedNamespaces[0]);
// Iterate ALL NVS entries and populate
nvs_iterator_t it = nullptr;
esp_err_t err = nvs_entry_find("nvs", nullptr, NVS_TYPE_ANY, &it);
if (err != ESP_OK) {
LOG_ERROR("Could not initialize NVS iterator (error %d - %s)", err, esp_err_to_name(err));
return false;
}
uint32_t entryCount = 0;
while (err == ESP_OK && entryCount < MAX_LOOP_ITERATIONS) {
nvs_entry_info_t info;
nvs_entry_info(it, &info);
entryCount++;
// Reset task watchdog periodically to prevent timeout during long NVS iteration
// This is safe because we're making progress, not stuck
if (entryCount % 10 == 0) {
esp_task_wdt_reset();
}
// Skip excluded namespaces
bool isExcluded = false;
for (size_t i = 0; i < excludedCount; i++) {
if (strcmp(info.namespace_name, excludedNamespaces[i]) == 0) {
isExcluded = true;
break;
}
}
if (isExcluded) {
err = nvs_entry_next(&it);
continue;
}
// Auto-create namespace object if first time seeing it
if (!doc["nvs"][info.namespace_name].is<JsonObject>()) {
doc["nvs"][info.namespace_name].to<JsonObject>();
}
// Read and store value based on type
Preferences prefs;
if (prefs.begin(info.namespace_name, true)) { // read-only
switch(info.type) {
case NVS_TYPE_U8:
doc["nvs"][info.namespace_name][info.key]["type"] = "u8";
doc["nvs"][info.namespace_name][info.key]["value"] = prefs.getUChar(info.key, 0);
break;
case NVS_TYPE_I8:
doc["nvs"][info.namespace_name][info.key]["type"] = "i8";
doc["nvs"][info.namespace_name][info.key]["value"] = prefs.getChar(info.key, 0);
break;
case NVS_TYPE_U16:
doc["nvs"][info.namespace_name][info.key]["type"] = "u16";
doc["nvs"][info.namespace_name][info.key]["value"] = prefs.getUShort(info.key, 0);
break;
case NVS_TYPE_I16:
doc["nvs"][info.namespace_name][info.key]["type"] = "i16";
doc["nvs"][info.namespace_name][info.key]["value"] = prefs.getShort(info.key, 0);
break;
case NVS_TYPE_U32:
doc["nvs"][info.namespace_name][info.key]["type"] = "u32";
doc["nvs"][info.namespace_name][info.key]["value"] = prefs.getUInt(info.key, 0);
break;
case NVS_TYPE_I32:
doc["nvs"][info.namespace_name][info.key]["type"] = "i32";
doc["nvs"][info.namespace_name][info.key]["value"] = prefs.getInt(info.key, 0);
break;
case NVS_TYPE_U64:
doc["nvs"][info.namespace_name][info.key]["type"] = "u64";
doc["nvs"][info.namespace_name][info.key]["value"] = prefs.getULong64(info.key, 0);
break;
case NVS_TYPE_I64:
doc["nvs"][info.namespace_name][info.key]["type"] = "i64";
doc["nvs"][info.namespace_name][info.key]["value"] = prefs.getLong64(info.key, 0);
break;
case NVS_TYPE_STR: {
char strBuffer[NVS_STRING_MAX_SIZE];
size_t strLen = prefs.getString(info.key, strBuffer, sizeof(strBuffer));
if (strLen > 0) {
doc["nvs"][info.namespace_name][info.key]["value"] = strBuffer;
} else {
doc["nvs"][info.namespace_name][info.key]["value"] = "";
}
doc["nvs"][info.namespace_name][info.key]["type"] = "str";
break;
}
case NVS_TYPE_BLOB: {
// Get blob size
size_t blobSize = prefs.getBytesLength(info.key);
if (blobSize > 0) {
// Allocate buffer for blob data
uint8_t* blobData = (uint8_t*)ps_malloc(blobSize);
if (blobData != nullptr) {
size_t readSize = prefs.getBytes(info.key, blobData, blobSize);
if (readSize > 0) {
// Try to interpret common blob sizes as numeric types
if (readSize == sizeof(float)) {
// Interpret as float
float value;
memcpy(&value, blobData, sizeof(float));
doc["nvs"][info.namespace_name][info.key]["value"] = value;
doc["nvs"][info.namespace_name][info.key]["type"] = "float";
} else if (readSize == sizeof(double)) {
// Interpret as double
double value;
memcpy(&value, blobData, sizeof(double));
doc["nvs"][info.namespace_name][info.key]["value"] = value;
doc["nvs"][info.namespace_name][info.key]["type"] = "double";
} else {
// Not supported since it makes the JSON too large
LOG_WARNING("Skipping blob key %s in namespace %s due to unsupported size %zu bytes",
info.key, info.namespace_name, readSize);
}
}
free(blobData);
}
}
break;
}
default:
LOG_WARNING("Unknown NVS type %d for key %s in namespace %s", info.type, info.key, info.namespace_name);
break;
}
prefs.end();
}
err = nvs_entry_next(&it);
if (err != ESP_OK && err != ESP_ERR_NVS_NOT_FOUND) {
LOG_ERROR("Could not advance NVS iterator (error %d - %s)", err, esp_err_to_name(err));
return false;
}
}
if (it != nullptr) {
nvs_release_iterator(it);
}
LOG_DEBUG("Completed exporting %u NVS entries to JSON. Size of JSON: %zu bytes", entryCount, measureJson(doc));
return true;
}
// Background task that creates TAR data for streaming backup
static void tarPackerTask(void* param) {
RingBufferStream* stream = (RingBufferStream*)param;
LOG_DEBUG("TAR packing task started");
// Collect all files and directories from root
std::vector<TAR::dir_entity_t> dirEntities;
TarPacker::collectDirEntities(&dirEntities, &LittleFS, "/");
if (dirEntities.empty()) {
LOG_ERROR("No files found in LittleFS for backup");
stream->setError();
vTaskDelete(NULL);
return;
}
LOG_DEBUG("Collected %d entities for TAR packing", dirEntities.size());
// Set progress callback to feed watchdog during packing
TarPacker::setProgressCallBack([](size_t packedBytes, size_t totalBytes) {
const size_t progressInterval = 1024 * 32; // Reset watchdog every 32KB
if (packedBytes % progressInterval == 0 || packedBytes == totalBytes) {
esp_task_wdt_reset();
LOG_DEBUG("TAR packing progress: %zu / %zu bytes (%.2f%%)",
packedBytes, totalBytes, (double)packedBytes / totalBytes * 100);
}
});
LOG_DEBUG("Starting TAR packing to stream");
size_t packed = TarPacker::pack_files(&LittleFS, dirEntities, stream);
// Clear progress callback
TarPacker::setProgressCallBack(nullptr);
if (packed > 0) {
LOG_INFO("TAR packing complete: %zu bytes", packed);
stream->setEOF(); // Signal consumer we're done
} else {
LOG_ERROR("TAR packing failed (error code: %zu)", packed);
stream->setError();
}
LOG_DEBUG("Tar packer task exiting");
vTaskDelete(NULL); // Delete self
}
// Start streaming backup (returns stream that caller must delete when done)
RingBufferStream* startStreamingBackup() {
LOG_DEBUG("Starting streaming backup");
RingBufferStream* stream = new RingBufferStream();
if (!stream) {
LOG_ERROR("Failed to start streaming backup");
return nullptr;
}
BaseType_t result = xTaskCreate(
tarPackerTask,
TASK_TAR_PACKER_NAME,
TASK_TAR_PACKER_STACK_SIZE,
stream,
TASK_TAR_PACKER_PRIORITY,
nullptr
);
if (result != pdPASS) {
LOG_ERROR("Failed to create tar packer task");
delete stream;
return nullptr;
}
LOG_DEBUG("Streaming backup started successfully");
return stream;
}
// ========== RESTORE UTILITIES ==========
// Check if configuration restore is pending (set by restore endpoint before restart)
bool isNvsRestorePending() {
Preferences prefs;
if (!prefs.begin(PREFERENCES_NAMESPACE_GENERAL, true)) {
return false;
}
bool pending = prefs.getBool("restore_pending", false);
prefs.end();
return pending;
}
// Restore NVS from JSON document (inverse of nvsDataToJson)
bool restoreNvsFromJson(JsonDocument &doc) {
LOG_INFO("Starting NVS configuration restore");
uint32_t totalKeys = 0;
uint32_t successKeys = 0;
uint32_t failedKeys = 0;
// Iterate namespaces
for (JsonPair nsPair : doc["nvs"].as<JsonObject>()) {
const char* ns = nsPair.key().c_str();
// Skip excluded namespaces (should not be present in restore file, but safety check)
const char* excludedNamespaces[] = {EXCLUDED_NVS_NAMESPACES_LIST};
bool isExcluded = false;
for (const char* excluded : excludedNamespaces) {
if (strcmp(ns, excluded) == 0) {
LOG_WARNING("Skipping excluded namespace in restore: %s (security/device-specific data)", ns);
isExcluded = true;
break;
}
}
if (isExcluded) continue;
// A backup taken on a product with Ethernet carries eth_ns. That namespace must never
// exist on a product without it: its commissioning marker alone would close the
// provisioning carve-out of a WiFi-only device.
if (strcmp(ns, PREFERENCES_NAMESPACE_ETH) == 0 && !globalHwProfile->hasEthernet) {
LOG_WARNING("Skipping namespace in restore: %s (this product has no Ethernet)", ns);
continue;
}
LOG_DEBUG("Restoring namespace: %s", ns);
Preferences prefs;
if (!prefs.begin(ns, false)) {
LOG_WARNING("Failed to open namespace: %s", ns);
continue;
}
// Iterate keys in namespace
for (JsonPair keyPair : nsPair.value().as<JsonObject>()) {
const char* key = keyPair.key().c_str();
JsonVariant keyData = keyPair.value();
totalKeys++;
bool success = false;
// Read type and value from new format: {"type": "u8", "value": 123}
if (!keyData.is<JsonObject>() || !keyData["type"].is<const char*>() || !keyData.containsKey("value")) {
LOG_WARNING("Invalid key format: %s/%s (missing type or value)", ns, key);
failedKeys++;
continue;
}
const char* typeStr = keyData["type"].as<const char*>();
JsonVariant valueVar = keyData["value"];
// Write based on explicit type metadata (no guessing needed!)
if (strcmp(typeStr, "u8") == 0) {
success = prefs.putUChar(key, (uint8_t)valueVar.as<int>());
} else if (strcmp(typeStr, "i8") == 0) {
success = prefs.putChar(key, (int8_t)valueVar.as<int>());
} else if (strcmp(typeStr, "u16") == 0) {
success = prefs.putUShort(key, (uint16_t)valueVar.as<int>());
} else if (strcmp(typeStr, "i16") == 0) {
success = prefs.putShort(key, (int16_t)valueVar.as<int>());
} else if (strcmp(typeStr, "u32") == 0) {
success = prefs.putUInt(key, (uint32_t)valueVar.as<unsigned long>());
} else if (strcmp(typeStr, "i32") == 0) {
success = prefs.putInt(key, (int32_t)valueVar.as<long>());
} else if (strcmp(typeStr, "u64") == 0) {
success = prefs.putULong64(key, (uint64_t)valueVar.as<unsigned long long>());
} else if (strcmp(typeStr, "i64") == 0) {
success = prefs.putLong64(key, (int64_t)valueVar.as<long long>());
} else if (strcmp(typeStr, "float") == 0) {
success = prefs.putFloat(key, valueVar.as<float>());
} else if (strcmp(typeStr, "str") == 0) {
size_t len = prefs.putString(key, valueVar.as<const char*>()); // This returns the length of the string stored, not true/false
success = strlen(valueVar.as<const char*>()) == len;
} else if (strcmp(typeStr, "blob") == 0) {
LOG_WARNING("BLOB type not yet supported during restore: %s/%s", ns, key);
failedKeys++;
continue;
} else {
LOG_WARNING("Unknown NVS type '%s' for key %s/%s", typeStr, ns, key);
failedKeys++;
continue;
}
if (success) {
successKeys++;
} else {
failedKeys++;
LOG_WARNING("Failed to restore key: %s/%s (type: %s)", ns, key, typeStr);
}
}
prefs.end();
}
LOG_INFO("NVS restore complete: %lu/%lu keys succeeded, %lu failed",
successKeys, totalKeys, failedKeys);
return totalKeys > 0 && successKeys > 0;
}
// Perform configuration restore from staged file (called during early boot, before services start)
void performNvsRestore() {
LOG_INFO("Configuration restore pending flag detected");
// Clear flag immediately to prevent retry loops on failure
Preferences prefs;
if (prefs.begin(PREFERENCES_NAMESPACE_GENERAL, false)) {
prefs.putBool("restore_pending", false);
prefs.end();
}
// Check if restore file exists
if (!LittleFS.exists("/restore/nvs_restore.json")) {
LOG_ERROR("Restore file not found, skipping restore");
return;
}
// LED indicator: orange = restoring. Boot continues after this function, so a
// critical layer left occupied would mask the rest of the boot walk and the
// steady-state colour for the whole uptime. The guard releases it on every exit
// path, including any added later.
Led::setOrange(Led::PRIO_CRITICAL);
struct RestoreLedGuard {
~RestoreLedGuard() { Led::clearPattern(Led::PRIO_CRITICAL); }
} restoreLedGuard;
// Read and parse restore file
File restoreFile = LittleFS.open("/restore/nvs_restore.json", FILE_READ);
if (!restoreFile) {
LOG_ERROR("Failed to open restore file");
LittleFS.remove("/restore/nvs_restore.json");
return;
}
SpiRamAllocator allocator;
JsonDocument doc(&allocator);
DeserializationError error = deserializeJson(doc, restoreFile);
restoreFile.close();
if (error) {
LOG_ERROR("Failed to parse restore JSON: %s", error.c_str());
LittleFS.remove("/restore/nvs_restore.json");
return;
}
// Perform restore
bool success = restoreNvsFromJson(doc);
// Clean up restore file
LittleFS.remove("/restore/nvs_restore.json");
// Ahead of the guard, not left to it: the outcome colour below is shown during
// the feedback delay, which is still inside this scope.
Led::clearPattern(Led::PRIO_CRITICAL);
if (success) {
LOG_INFO("Configuration restored successfully");
Led::setGreen(Led::PRIO_NORMAL);
} else {
LOG_ERROR("Configuration restore failed or incomplete");
Led::setRed(Led::PRIO_URGENT);
}
delay(2000); // Brief visual feedback
}
// Check if backup version is compatible with current firmware
// Compatibility rule: backup can be restored if:
// - Backup major version == current major version
// - Backup version <= current version
// This allows forward compatibility (1.0.0 backup on 1.5.0 firmware) but not backward (1.5.0 backup on 1.0.0)
bool isBackupVersionCompatible(const char* backupVersion) {
if (!backupVersion) {
LOG_WARNING("Backup version is null, cannot check compatibility");
return false;
}
// Parse backup version (format: "X.Y.Z" or "X.Y.Z (dev)")
int backupMajor = 0, backupMinor = 0, backupPatch = 0;
if (sscanf(backupVersion, "%d.%d.%d", &backupMajor, &backupMinor, &backupPatch) < 3) {
LOG_WARNING("Invalid backup version format: %s", backupVersion);
return false;
}
// Get current firmware version from build info (defined in constants.h)
// The version string format: "X.Y.Z"
const char* currentVersionStr = FIRMWARE_BUILD_VERSION;
int currentMajor = 0, currentMinor = 0, currentPatch = 0;
if (sscanf(currentVersionStr, "%d.%d.%d", ¤tMajor, ¤tMinor, ¤tPatch) < 3) {
LOG_WARNING("Invalid current firmware version format: %s", currentVersionStr);
return false;
}
LOG_DEBUG("Version check - Backup: %d.%d.%d, Current: %d.%d.%d",
backupMajor, backupMinor, backupPatch, currentMajor, currentMinor, currentPatch);
// Check major version match
if (backupMajor != currentMajor) {
LOG_WARNING("Version incompatible: backup major version %d != current major version %d",
backupMajor, currentMajor);
return false;
}
// Check if backup version <= current version
if (backupMajor < currentMajor) return true; // Should not reach here due to earlier check
if (backupMinor > currentMinor) {
LOG_WARNING("Version incompatible: backup minor version %d > current minor version %d",
backupMinor, currentMinor);
return false;
}
if (backupMinor == currentMinor && backupPatch > currentPatch) {
LOG_WARNING("Version incompatible: backup patch version %d > current patch version %d",
backupPatch, currentPatch);
return false;
}
LOG_DEBUG("Backup version compatible: %d.%d.%d <= %d.%d.%d (same major)",
backupMajor, backupMinor, backupPatch, currentMajor, currentMinor, currentPatch);
return true;
}
// Compare two "vX.Y.Z" or "X.Y.Z" version strings
// Returns >0 if current is newer than available, 0 if equal, <0 if current is older
// Null or unparseable input degrades to "0.0.0" - see lib/version_compare for the
// (unit-tested) implementation and its null/malformed-input guarantees.
int compareVersions(const char* current, const char* available) {
return VersionCompare::compare(current, available);
}f4329a85eThis file is pinned to the selected release. Download the original to open it in your design tool.