Can not HTTP GET to the full information in wokwi-cli

[crjssy]

Hi！ @urish

I'm trying to use wokwi cli now to use http get ip-location, but the information I get is incomplete, can you check it?
The code as follows:

#include "esp32-ip-to-geolocation.h"
#include <string.h>

#define MAX_HTTP_OUTPUT_BUFFER 8192

char *output_buffer; // Buffer to store HTTP response
int output_len;      // Stores number of bytes in output_buffer

// Global event group for WiFi status
EventGroupHandle_t s_wifi_event_group;
const int WIFI_CONNECTED_BIT = BIT0;
const int WIFI_FAIL_BIT = BIT1;

void event_handler(void *arg, esp_event_base_t event_base, int32_t event_id, void *event_data)
{
    if (event_base == WIFI_EVENT && event_id == WIFI_EVENT_STA_START)
    {
        esp_wifi_connect();
    }
    else if (event_base == WIFI_EVENT && event_id == WIFI_EVENT_STA_DISCONNECTED)
    {
        esp_wifi_connect();
        ESP_LOGI(TAG, "retry to connect to the AP");
        xEventGroupSetBits(s_wifi_event_group, WIFI_FAIL_BIT);
    }
    else if (event_base == IP_EVENT && event_id == IP_EVENT_STA_GOT_IP)
    {
        ip_event_got_ip_t *event = (ip_event_got_ip_t *)event_data;
        ESP_LOGI(TAG, "got ip:%s", ip4addr_ntoa((ip4_addr_t *)&event->ip_info.ip));
        xEventGroupSetBits(s_wifi_event_group, WIFI_CONNECTED_BIT);
    }
}

void wifi_init_sta(void)
{
    s_wifi_event_group = xEventGroupCreate();
    ESP_ERROR_CHECK(esp_netif_init());
    ESP_ERROR_CHECK(esp_event_loop_create_default());
    esp_netif_create_default_wifi_sta();

    wifi_init_config_t cfg = WIFI_INIT_CONFIG_DEFAULT();
    ESP_ERROR_CHECK(esp_wifi_init(&cfg));

    esp_event_handler_instance_t instance_any_id;
    esp_event_handler_instance_t instance_got_ip;
    ESP_ERROR_CHECK(esp_event_handler_instance_register(WIFI_EVENT, ESP_EVENT_ANY_ID, &event_handler, NULL, &instance_any_id));
    ESP_ERROR_CHECK(esp_event_handler_instance_register(IP_EVENT, IP_EVENT_STA_GOT_IP, &event_handler, NULL, &instance_got_ip));

    wifi_config_t wifi_config = {
        .sta = {
            .ssid = CONFIG_WIFI_SSID,
            .password = CONFIG_WIFI_PASSWORD,
            .threshold.authmode = CONFIG_WIFI_AUTH_MODE,
        },
    };

    ESP_ERROR_CHECK(esp_wifi_set_mode(WIFI_MODE_STA));
    ESP_ERROR_CHECK(esp_wifi_set_config(ESP_IF_WIFI_STA, &wifi_config));
    ESP_ERROR_CHECK(esp_wifi_start());

    ESP_LOGI(TAG, "wifi_init_sta finished.");

    EventBits_t bits = xEventGroupWaitBits(s_wifi_event_group, WIFI_CONNECTED_BIT | WIFI_FAIL_BIT, pdFALSE, pdFALSE, portMAX_DELAY);

    if (bits & WIFI_CONNECTED_BIT)
    {
        ESP_LOGI(TAG, "connected to ap SSID:%s", CONFIG_WIFI_SSID);
    }
    else if (bits & WIFI_FAIL_BIT)
    {
        ESP_LOGI(TAG, "Failed to connect to SSID:%s", CONFIG_WIFI_SSID);
    }
    else
    {
        ESP_LOGE(TAG, "UNEXPECTED EVENT");
    }
}

char *output_buffer = NULL; // Buffer to store HTTP response
int output_len = 0; // Stores number of bytes in output_buffer

esp_err_t http_event_handler(esp_http_client_event_t *evt)
{
    switch (evt->event_id)
    {
    case HTTP_EVENT_ERROR:
        ESP_LOGI(TAG, "HTTP_EVENT_ERROR");
        break;
    case HTTP_EVENT_ON_CONNECTED:
        ESP_LOGI(TAG, "HTTP_EVENT_ON_CONNECTED");
        break;
    case HTTP_EVENT_HEADER_SENT:
        ESP_LOGI(TAG, "HTTP_EVENT_HEADER_SENT");
        break;
    case HTTP_EVENT_ON_HEADER:
        ESP_LOGI(TAG, "HTTP_EVENT_ON_HEADER, key=%s, value=%s", evt->header_key, evt->header_value);
        break;
    case HTTP_EVENT_ON_DATA:
        ESP_LOGI(TAG, "HTTP_EVENT_ON_DATA, len=%d", evt->data_len);
        if (evt->data_len > 0)
        {
            if (output_buffer == NULL)
            {
                // Allocate memory for the output buffer
                output_buffer = (char *)malloc(evt->data_len + 1);
                if (output_buffer == NULL)
                {
                    ESP_LOGE(TAG, "Failed to allocate memory for output buffer");
                    return ESP_FAIL;
                }
                output_len = 0;
            }
            else
            {
                // Reallocate memory for the output buffer to accommodate new data
                char *new_buffer = (char *)realloc(output_buffer, output_len + evt->data_len + 1);
                if (new_buffer == NULL)
                {
                    ESP_LOGE(TAG, "Failed to reallocate memory for output buffer");
                    free(output_buffer);
                    output_buffer = NULL;
                    output_len = 0;
                    return ESP_FAIL;
                }
                output_buffer = new_buffer;
            }

            // Copy the new data into the buffer
            memcpy(output_buffer + output_len, evt->data, evt->data_len);
            output_len += evt->data_len;
            output_buffer[output_len] = '\0'; // Null-terminate the buffer
        }
        break;
    case HTTP_EVENT_ON_FINISH:
        ESP_LOGI(TAG, "HTTP_EVENT_ON_FINISH");
        if (output_buffer != NULL)
        {
            // Parse the JSON data
            cJSON *json = cJSON_Parse(output_buffer);
            if (json == NULL)
            {
                ESP_LOGE(TAG, "JSON parsing error");
            }
            else
            {
                const char *keys[] = {
                    "status", "country", "countryCode", "region", "regionName", "city",
                    "zip", "lat", "lon", "timezone", "isp", "org", "as", "query"};
                int numKeys = sizeof(keys) / sizeof(keys[0]);
                for (int i = 0; i < numKeys; i++)
                {
                    cJSON *value = cJSON_GetObjectItemCaseSensitive(json, keys[i]);
                    if (cJSON_IsString(value) && (value->valuestring != NULL))
                    {
                        ESP_LOGI(TAG, "%s: %s", keys[i], value->valuestring);
                    }
                    else if (cJSON_IsNumber(value))
                    {
                        ESP_LOGI(TAG, "%s: %f", keys[i], value->valuedouble);
                    }
                }

                cJSON_Delete(json);
            }
            // Free the output buffer
            free(output_buffer);
            output_buffer = NULL;
            output_len = 0;
        }
        break;
    case HTTP_EVENT_DISCONNECTED:
        ESP_LOGI(TAG, "HTTP_EVENT_DISCONNECTED");
        if (output_buffer != NULL)
        {
            free(output_buffer);
            output_buffer = NULL;
            output_len = 0;
        }
        break;
    case HTTP_EVENT_REDIRECT:
        ESP_LOGI(TAG, "HTTP_EVENT_REDIRECT");
        break;
    default:
        ESP_LOGI(TAG, "Unhandled HTTP event: %d", evt->event_id);
        break;
    }
    return ESP_OK;
}

void http_get_task(void *pvParameters)
{
    output_buffer = NULL;
    output_len = 0;

    esp_http_client_config_t config = {
        .url = URL,
        .event_handler = http_event_handler, 
        .method = HTTP_METHOD_GET
    };

    esp_http_client_handle_t client = esp_http_client_init(&config);
    esp_err_t err = esp_http_client_perform(client);

    if (err == ESP_OK)
    {
        int status = esp_http_client_get_status_code(client);
        int content_length = esp_http_client_get_content_length(client);
        ESP_LOGI(TAG, "HTTP GET Status = %d, content_length = %d", status, content_length);
    }
    else
    {
        ESP_LOGE(TAG, "HTTP GET request failed: %s", esp_err_to_name(err));
    }

    esp_http_client_cleanup(client);

    // Free the output buffer if it was allocated
    if (output_buffer != NULL)
    {
        free(output_buffer);
        output_buffer = NULL;
        output_len = 0;
    }
    vTaskDelete(NULL);
}


void app_main()
{
    esp_err_t ret = nvs_flash_init();
    if (ret == ESP_ERR_NVS_NO_FREE_PAGES || ret == ESP_ERR_NVS_NEW_VERSION_FOUND)
    {
        ESP_ERROR_CHECK(nvs_flash_erase());
        ret = nvs_flash_init();
    }
    ESP_ERROR_CHECK(ret);

    esp_log_level_set("*", ESP_LOG_DEBUG);

    wifi_init_sta();
    xTaskCreate(&http_get_task, "http_get_task", 8192, NULL, 5, NULL);
} 

My yml file is as follows:

name:` Build and Test Application

on:
  pull_request:
    branches:
      - main
  push:
    branches:
      - main
  workflow_dispatch:

env:
  IDF_PATH: /opt/esp/idf
  test_dirs: .

defaults:
  run:
    shell: bash

jobs:
  build-test-app:
    name: Build Test App
    strategy:
      fail-fast: false
      matrix:
        idf-branch:
          - release-v5.0
          - release-v5.1
        target:
          - esp32
    runs-on: ubuntu-22.04
    container:
      image: espressif/idf:${{ matrix.idf-branch }}
    steps:
      - uses: actions/checkout@v4
      - name: Install Python Dependencies
        run: |
          . $IDF_PATH/export.sh
          python -m pip install --upgrade pip
          python -m pip install idf-build-apps
      - name: Build Test Application with ESP-IDF
        run: |
          . $IDF_PATH/export.sh
          idf-build-apps build \
            -p ${{ env.test_dirs }} \
            --target ${{ matrix.target }} \
            --recursive \
            --build-dir build_${{ matrix.target }}_${{ matrix.idf-branch }} 
            idf.py -DSDKCONFIG=/__w/esp32-ip-to-geolocation/esp32-ip-to-geolocation/sdkconfig build > build_output.log 2>&1
      - name: Upload files to artifacts for run-target job
        uses: actions/upload-artifact@v4
        with:
          name: built_binaries_${{ matrix.target }}_${{ matrix.idf-branch }}
          path: |
            **/build**/bootloader/bootloader.bin
            **/build**/partition_table/partition-table.bin
            **/build**/*.bin
            **/build**/*.elf
            **/build**/flasher_args.json
          if-no-files-found: error

  simulate-test:
    name: Simulate Test on WokWi
    needs: build-test-app
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false
      matrix:
        idf-branch:
          - release-v5.0
          - release-v5.1
        target:
          - esp32
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.9'  # Ensure this matches the Python version needed for your testing
      - name: Download built binaries from build job
        uses: actions/download-artifact@v4
        with:
          name: built_binaries_${{ matrix.target }}_${{ matrix.idf-branch }}
          path: .
      - name: Install the Wokwi CLI
        run: curl -L https://wokwi.com/ci/install.sh | sh
      - name: Install Python packages for PyTest
        run: pip install -r requirements.txt
      - name: Run Test App in Wokwi Simulation and Save Serial Output
        env:
          WOKWI_CLI_TOKEN: ${{ secrets.WOKWI_CLI_TOKEN }}
        run: |
          pytest ${{ env.test_dirs }} \
            --embedded-services idf,wokwi \
            --tb short \
            --junit-xml test_wokwi_${{ matrix.target }}_${{ matrix.idf-branch }}.xml \
            | tee serial_output_${{ matrix.target }}_${{ matrix.idf-branch }}.txt
      - name: Upload Test Results and Serial Output
        uses: actions/upload-artifact@v4
        with:
          name: test_wokwi_${{ matrix.target }}_${{ matrix.idf-branch }}_junit
          path: |
            test_wokwi_${{ matrix.target }}_${{ matrix.idf-branch }}.xml
            serial_output_${{ matrix.target }}_${{ matrix.idf-branch }}.txt

  publish-results:
    name: Publish Test App results
    needs: simulate-test
    runs-on: ubuntu-20.04
    if: always() # (run even if the previous steps have failed)
    steps:
      - name: Download Test results
        uses: actions/download-artifact@v4
        with:
          path: test_results
      - name: Publish Test Results
        uses: EnricoMi/publish-unit-test-result-action@v2
        with:
          files: test_results/**/*.xml

My pytest as follows:

# SPDX-FileCopyrightText: 2022-2024 Hays Chan
# SPDX-License-Identifier: MIT

'''
Steps to run these cases:
- Build
  - . ${IDF_PATH}/export.sh
  - pip install idf_build_apps
  - python tools/build_apps.py components/button/test_apps -t esp32
- Test
  - pip install -r tools/requirements/requirement.pytest.txt
  - pytest components/button/test_apps --target esp32
'''

import pytest
from pytest_embedded import Dut

@pytest.mark.supported_targets("esp32")  # Specify the target, esp32 in this case
def test_esp32_ip_to_geolocation(dut: Dut):
    # Start the test
    dut.expect_exact("wifi_init_sta finished.")
    dut.expect("connected to ap SSID:Wokwi-GUEST")

    dut.expect("HTTP_EVENT_ON_FINISH")  

    # Check for the expected logs from the JSON response
    expected_keys = [
        "status:",
        "country:",
        "countryCode:",
        "region:",
        "regionName:",
        "city:",
        "zip:",
        "lat:",
        "lon:",
        "timezone",
        "isp",
        "org",
        "as",
        "query"
    ]

    #  # Check for a successful HTTP request
    # dut.expect("HTTP GET Status = ", timeout=120)  

    # Check each expected log entry for presence only, not specific content
    for key in expected_keys:
        dut.expect(key)

It doesn't output the full message I want,you can check out my action

https://github.com/crjssy/esp32-ip-to-geolocation/actions/runs/9557963622

Any help on this would be really appreciated! 🙋

[urish]

Hello, thanks for the detailed report!

Does the same firmware work differently on Wokwi for VS Code? Or do you also get the same result there?

[crjssy]

The same firmware work on Wokwi for VS Code like this

invalid header: 0x72656461

[crjssy]

The same firmware work on Wokwi for VS Code like this

invalid header: 0x72656461

Thank you for your reply ,I build and open simulator it displays an error like this ⬆️

And I try to add timeout in pytest the v5.1 can pass the test, but it's only an occasional occurrence.

# SPDX-FileCopyrightText: 2022-2024 Hays Chan
# SPDX-License-Identifier: MIT

'''
Steps to run these cases:
- Build
  - . ${IDF_PATH}/export.sh
  - pip install idf_build_apps
  - python tools/build_apps.py components/button/test_apps -t esp32
- Test
  - pip install -r tools/requirements/requirement.pytest.txt
  - pytest components/button/test_apps --target esp32
'''

import pytest
from pytest_embedded import Dut

@pytest.mark.supported_targets("esp32")
def test_esp32_ip_to_geolocation(dut: Dut):
    # Start the test
    dut.expect_exact("wifi_init_sta finished.", timeout=60)
    dut.expect("connected to ap SSID:Wokwi-GUEST", timeout=60)

    dut.expect("HTTP_EVENT_ON_FINISH", timeout=60)

    # Check for the expected logs from the JSON response
    expected_keys = [
        "status:",
        "country:",
        "countryCode:",
        "region:",
        "regionName:",
        "city:",
        "zip:",
        "lat:",
        "lon:",
        "timezone",
        "isp",
        "org",
        "as",
        "query"
    ]

    # Check each expected log entry for presence only, not specific content
    for key in expected_keys:
        dut.expect(key, timeout=60)

https://github.com/crjssy/esp32-ip-to-geolocation/actions/runs/9559851466