Merge branch 'main' into remove-setenv

.gitattributes

@@ -0,0 +1,5 @@
+# Set the default behavior, in case people don't have core.autocrlf set.
+* text=auto
+
+# Declare files that will always have CRLF line endings on checkout.
+tests/resources/testparse_crlf.crt text eol=crlf

.github/workflows/ci.yml

@@ -6,7 +6,7 @@ on:
       - 'main'
 
 env:
-  BUILDER_VERSION: v0.9.40
+  BUILDER_VERSION: v0.9.57
   BUILDER_SOURCE: releases
   BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net
   PACKAGE_NAME: aws-c-io
@@ -18,7 +18,7 @@ env:
 
 jobs:
   linux-compat:
-    runs-on: ubuntu-20.04 # latest
+    runs-on: ubuntu-22.04 # latest
     strategy:
       fail-fast: false
       matrix:
@@ -38,7 +38,7 @@ jobs:
         ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }}
 
   linux-byo-crypto:
-    runs-on: ubuntu-20.04 # latest
+    runs-on: ubuntu-22.04 # latest
     steps:
        # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages
     - name: Build ${{ env.PACKAGE_NAME }} + consumers
@@ -47,7 +47,7 @@ jobs:
         ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-al2-x64 build -p ${{ env.PACKAGE_NAME }} --cmake-extra=-DBYO_CRYPTO=ON
 
   linux-compiler-compat:
-    runs-on: ubuntu-20.04 # latest
+    runs-on: ubuntu-22.04 # latest
     strategy:
       matrix:
         compiler:
@@ -70,7 +70,7 @@ jobs:
         ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --compiler=${{ matrix.compiler }}
 
   clang-sanitizers:
-    runs-on: ubuntu-20.04 # latest
+    runs-on: ubuntu-22.04 # latest
     strategy:
       matrix:
         sanitizers: [",thread", ",address,undefined"]
@@ -82,7 +82,7 @@ jobs:
           ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --compiler=clang-11 --cmake-extra=-DENABLE_SANITIZERS=ON --cmake-extra=-DSANITIZERS="${{ matrix.sanitizers }}"
 
   linux-shared-libs:
-    runs-on: ubuntu-20.04 # latest
+    runs-on: ubuntu-22.04 # latest
     steps:
         # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages
     - name: Build ${{ env.PACKAGE_NAME }}
@@ -93,7 +93,7 @@ jobs:
   # Test downstream repos.
   # This should not be required because we can run into a chicken and egg problem if there is a change that needs some fix in a downstream repo.
   downstream:
-    runs-on: ubuntu-20.04 # latest
+    runs-on: ubuntu-22.04 # latest
     steps:
       # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages
       - name: Build ${{ env.PACKAGE_NAME }}
@@ -140,10 +140,48 @@ jobs:
         python .\aws-c-io\build\deps\aws-c-common\scripts\appverifier_ctest.py --build_directory .\aws-c-io\build\aws-c-io
 
   osx:
-    runs-on: macos-12 # latest
+    runs-on: macos-13 # latest
     steps:
     - name: Build ${{ env.PACKAGE_NAME }} + consumers
       run: |
         python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
         chmod a+x builder
         ./builder build -p ${{ env.PACKAGE_NAME }}
+
+  freebsd:
+    runs-on: ubuntu-22.04  # latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build ${{ env.PACKAGE_NAME }} + consumers
+        id: test
+        uses: cross-platform-actions/[email protected]
+        with:
+          operating_system: freebsd
+          architecture: x86-64
+          version: '14.0'
+          cpu_count: 4
+          shell: bash
+          run: |
+            sudo pkg install -y python3 py39-urllib3
+            python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
+            chmod a+x builder
+            ./builder build -p ${{ env.PACKAGE_NAME }}
+
+  openbsd:
+    runs-on: ubuntu-22.04 # latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build ${{ env.PACKAGE_NAME }} + consumers
+        id: test
+        uses: cross-platform-actions/[email protected]
+        with:
+          operating_system: openbsd
+          architecture: x86-64
+          version: '7.4'
+          cpu_count: 4
+          shell: bash
+          run: |
+            sudo pkg_add py3-urllib3
+            python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
+            chmod a+x builder
+            ./builder build -p ${{ env.PACKAGE_NAME }}

.github/workflows/handle-stale-discussions.yml

@@ -0,0 +1,18 @@
+name: HandleStaleDiscussions
+on:
+  schedule:
+    - cron: '0 */4 * * *'
+  discussion_comment:
+    types: [created]
+
+jobs:
+  handle-stale-discussions:
+    name: Handle stale discussions
+    runs-on: ubuntu-latest
+    permissions:
+      discussions: write
+    steps:
+      - name: Stale discussions action
+        uses: aws-github-ops/handle-stale-discussions@v1
+        env:
+          GITHUB_TOKEN:  ${{secrets.GITHUB_TOKEN}}

.github/workflows/proof-alarm.yml

@@ -16,7 +16,7 @@ jobs:
     - name: Check
       run: |
         TMPFILE=$(mktemp)
-        echo "8391c38acd85cca2a2892d02ebfd4ceb  source/linux/epoll_event_loop.c" > $TMPFILE
+        echo "c624a28de5af7f851a240a1e65a26c01  source/linux/epoll_event_loop.c" > $TMPFILE
         md5sum --check $TMPFILE
 
     # No further steps if successful
@@ -26,4 +26,5 @@ jobs:
       run: |
           echo "The VCC proofs are based on a snapshot of epoll_event_loop.c.
                 This push updates this file so the proofs must be rechecked to ensure they remain valid.
-                Please contact Nathan Chong."
+                Please contact Nathan Chong.
+                You can also update md5sum value by running `md5sum source/linux/epoll_event_loop.c` if the changes are trivial."

.github/workflows/stale_issue.yml

@@ -9,6 +9,9 @@ jobs:
   cleanup:
     runs-on: ubuntu-latest
     name: Stale issue job
+    permissions:
+      issues: write
+      pull-requests: write
     steps:
     - uses: aws-actions/stale-issue-cleanup@v3
       with:
@@ -32,7 +35,7 @@ jobs:
         # Issue timing
         days-before-stale: 2
         days-before-close: 5
-        days-before-ancient: 365
+        days-before-ancient: 36500
 
         # If you don't want to mark a issue as being ancient based on a
         # threshold of "upvotes", you can set this here. An "upvote" is

.gitignore

@@ -1,6 +1,7 @@
 # IDE Artifacts
 .metadata
 .build
+.vscode
 .idea
 *.d
 Debug

CMakeLists.txt

@@ -95,7 +95,7 @@ if (WIN32)
     #platform libs come from aws-c-common transitively, so we don't specify them here, but for documentation purposes,
     #Kernel32 and wsock2 are pulled in automatically. Here we add the lib containing the schannel API.
     #Also note, you don't get a choice on TLS implementation for Windows.
-    set(PLATFORM_LIBS Secur32 Crypt32)
+    set(PLATFORM_LIBS secur32 crypt32)
 elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" OR CMAKE_SYSTEM_NAME STREQUAL "Android")
     option(USE_VSOCK
 	    "Build in support for VSOCK sockets"
@@ -177,7 +177,7 @@ file(GLOB IO_SRC
 add_library(${PROJECT_NAME} ${LIBTYPE} ${IO_HEADERS} ${IO_SRC})
 aws_set_common_properties(${PROJECT_NAME})
 aws_prepare_symbol_visibility_args(${PROJECT_NAME} "AWS_IO")
-aws_check_headers(${PROJECT_NAME} ${IO_HEADERS})
+aws_check_headers(${PROJECT_NAME} ${AWS_IO_HEADERS})
 
 aws_add_sanitizers(${PROJECT_NAME})
 

NOTICE

@@ -1,3 +1,5 @@
 AWS C Io
 Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 SPDX-License-Identifier: Apache-2.0.
+
+The source/pkcs11/v2.40/pkcs11.h header file is based on Public Domain content from https://github.com/latchset/pkcs11-headers

README.md

@@ -208,7 +208,7 @@ Sockets interact directly with the underlying io and are invoked directly by the
 
 Platform | Implementation
 --- | ---
-Linux | Signal-to-noise (s2n) see github.com/awslabs/s2n
+Linux | Signal-to-noise (s2n) see: https://github.com/aws/s2n-tls
 BSD Variants | s2n
 Apple Devices | Security Framework/ Secure Transport. See https://developer.apple.com/documentation/security/secure_transport
 Windows | Secure Channel. See https://msdn.microsoft.com/en-us/library/windows/desktop/aa380123(v=vs.85).aspx

builder.json

@@ -5,8 +5,7 @@
         { "name": "aws-c-cal" },
         {
             "name": "s2n",
-            "revision": "v1.3.11",
-            "targets": ["linux", "android"]
+            "targets": ["linux", "android", "freebsd", "openbsd"]
         }
     ],
     "downstream": [

codebuild/linux-integration-tests.yml

@@ -10,8 +10,9 @@ phases:
   pre_build:
     commands:
       - export CC=gcc-7
-      - export BUILDER_VERSION=v0.9.29
-      - export BUILDER_SOURCE=releases
+      - export BUILDER_VERSION=$(cat .github/workflows/ci.yml | grep 'BUILDER_VERSION:' | sed 's/\s*BUILDER_VERSION:\s*\(.*\)/\1/')
+      - export BUILDER_SOURCE=$(cat .github/workflows/ci.yml | grep 'BUILDER_SOURCE:' | sed 's/\s*BUILDER_SOURCE:\s*\(.*\)/\1/')
+      - echo "Using builder version='${BUILDER_VERSION}' source='${BUILDER_SOURCE}'"
       - export BUILDER_HOST=https://d19elf31gohf1l.cloudfront.net
   build:
     commands:

include/aws/io/async_stream.h

@@ -0,0 +1,117 @@
+/**
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+
+#ifndef AWS_IO_ASYNC_STREAM_H
+#define AWS_IO_ASYNC_STREAM_H
+
+/**
+ * THIS IS AN EXPERIMENTAL AND UNSTABLE API
+ * TODO: logging
+ * TODO: modify API to return byte-bufs, instead of filling in the provided byte-buf?
+ *       this would avoid a copy in the use-cases we know of, but it's more complex
+ * TODO: vtable acquire()/release()?
+ * TODO: protect against simultaneous reads?
+ * TODO: check results of vtable->read() (i.e. 0 byte reads not allowed)?
+ *       this would require 1 or 2 additional allocations per read
+ */
+
+#include <aws/io/io.h>
+
+#include <aws/common/ref_count.h>
+
+AWS_PUSH_SANE_WARNING_LEVEL
+
+struct aws_async_input_stream;
+struct aws_byte_buf;
+struct aws_future_bool;
+struct aws_input_stream;
+
+struct aws_async_input_stream {
+    const struct aws_async_input_stream_vtable *vtable;
+    struct aws_allocator *alloc;
+    struct aws_ref_count ref_count;
+    void *impl;
+};
+
+struct aws_async_input_stream_vtable {
+    /**
+     * Destroy the stream, its refcount has reached 0.
+     */
+    void (*destroy)(struct aws_async_input_stream *stream);
+
+    /**
+     * Read once into the buffer.
+     * Complete the read when at least 1 byte is read, the buffer is full, or EOF is reached.
+     * Do not resize the buffer (do not use "aws_byte_buf_xyz_dynamic()" functions)
+     * Do not assume that buffer len starts at 0.
+     * You may assume that read() won't be called again until the current one completes.
+     * You may assume that the buffer has some space available.
+     * Return a future, which will contain an error code if something went wrong,
+     * or a result bool indicating whether EOF has been reached.
+     */
+    struct aws_future_bool *(*read)(struct aws_async_input_stream *stream, struct aws_byte_buf *dest);
+};
+
+AWS_EXTERN_C_BEGIN
+
+/**
+ * Initialize aws_async_input_stream "base class"
+ */
+AWS_IO_API
+void aws_async_input_stream_init_base(
+    struct aws_async_input_stream *stream,
+    struct aws_allocator *alloc,
+    const struct aws_async_input_stream_vtable *vtable,
+    void *impl);
+
+/**
+ * Increment reference count.
+ * You may pass in NULL (has no effect).
+ * Returns whatever pointer was passed in.
+ */
+AWS_IO_API
+struct aws_async_input_stream *aws_async_input_stream_acquire(struct aws_async_input_stream *stream);
+
+/**
+ * Decrement reference count.
+ * You may pass in NULL (has no effect).
+ * Always returns NULL.
+ */
+AWS_IO_API
+struct aws_async_input_stream *aws_async_input_stream_release(struct aws_async_input_stream *stream);
+
+/**
+ * Read once from the async stream into the buffer.
+ * The read completes when at least 1 byte is read, the buffer is full, or EOF is reached.
+ * Depending on implementation, the read could complete at any time.
+ * It may complete synchronously. It may complete on another thread.
+ * Returns a future, which will contain an error code if something went wrong,
+ * or a result bool indicating whether EOF has been reached.
+ *
+ * WARNING: The buffer must have space available.
+ * WARNING: Do not read again until the previous read is complete.
+ */
+AWS_IO_API
+struct aws_future_bool *aws_async_input_stream_read(struct aws_async_input_stream *stream, struct aws_byte_buf *dest);
+
+/**
+ * Read repeatedly from the async stream until the buffer is full, or EOF is reached.
+ * Depending on implementation, this could complete at any time.
+ * It may complete synchronously. It may complete on another thread.
+ * Returns a future, which will contain an error code if something went wrong,
+ * or a result bool indicating whether EOF has been reached.
+ *
+ * WARNING: The buffer must have space available.
+ * WARNING: Do not read again until the previous read is complete.
+ */
+AWS_IO_API
+struct aws_future_bool *aws_async_input_stream_read_to_fill(
+    struct aws_async_input_stream *stream,
+    struct aws_byte_buf *dest);
+
+AWS_EXTERN_C_END
+AWS_POP_SANE_WARNING_LEVEL
+
+#endif /* AWS_IO_ASYNC_STREAM_H */