Migrate generator to split core packages

BUILD

@@ -1,2 +1,2 @@
 # This is needed for mypy to be able to typecheck the tests properly
-python_requirements(name="test-reqs", source="requirements-dev.txt")
+python_requirements(name="test-reqs", source="requirements-dev.txt")

codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/ApplicationProtocol.java

@@ -53,9 +53,9 @@ public static ApplicationProtocol createDefaultHttpApplicationProtocol() {
     }
 
     private static Symbol createHttpSymbol(String symbolName) {
-        PythonDependency dependency = SmithyPythonDependency.SMITHY_PYTHON;
+        PythonDependency dependency = SmithyPythonDependency.SMITHY_HTTP;
         return Symbol.builder()
-                .namespace(dependency.packageName() + ".interfaces.http", ".")
+                .namespace(dependency.packageName() + ".aio.interfaces", ".")
                 .name(symbolName)
                 .addDependency(dependency)
                 .build();

codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/ClientGenerator.java

@@ -125,12 +125,12 @@ private void generateOperationExecutor(PythonWriter writer) {
         writer.addStdlibImport("copy", "deepcopy");
         writer.addStdlibImport("asyncio", "sleep");
 
-        writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON);
-        writer.addImport("smithy_python.exceptions", "SmithyRetryException");
-        writer.addImport("smithy_python.interfaces.interceptor", "Interceptor");
-        writer.addImport("smithy_python.interfaces.interceptor", "InterceptorContext");
-        writer.addImport("smithy_python.interfaces.retries", "RetryErrorInfo");
-        writer.addImport("smithy_python.interfaces.retries", "RetryErrorType");
+        writer.addDependency(SmithyPythonDependency.SMITHY_CORE);
+        writer.addImport("smithy_core.exceptions", "SmithyRetryException");
+        writer.addImport("smithy_core.interceptors", "Interceptor");
+        writer.addImport("smithy_core.interceptors", "InterceptorContext");
+        writer.addImport("smithy_core.interfaces.retries", "RetryErrorInfo");
+        writer.addImport("smithy_core.interfaces.retries", "RetryErrorType");
 
         writer.indent();
         writer.write("""
@@ -309,9 +309,10 @@ async def _handle_attempt(
                 writer.consumer(this::initializeHttpAuthParameters));
             writer.popState();
 
-            writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON);
-            writer.addImport("smithy_python.interfaces.identity", "Identity");
-            writer.addImports("smithy_python.interfaces.auth", Set.of("HTTPSigner", "HTTPAuthOption"));
+            writer.addDependency(SmithyPythonDependency.SMITHY_CORE);
+            writer.addDependency(SmithyPythonDependency.SMITHY_HTTP);
+            writer.addImport("smithy_core.interfaces.identity", "Identity");
+            writer.addImports("smithy_http.aio.interfaces.auth", Set.of("HTTPSigner", "HTTPAuthOption"));
             writer.addStdlibImport("typing", "Any");
             writer.write("""
                         auth_options = config.http_auth_scheme_resolver.resolve_auth_scheme(
@@ -345,10 +346,11 @@ async def _handle_attempt(
 
         writer.pushState(new ResolveEndpointSection());
         if (context.applicationProtocol().isHttpProtocol()) {
-            writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON);
+            writer.addDependency(SmithyPythonDependency.SMITHY_CORE);
+            writer.addDependency(SmithyPythonDependency.SMITHY_HTTP);
             // TODO: implement the endpoints 2.0 spec and remove the hard-coded handling of static params.
-            writer.addImport("smithy_python._private.http", "StaticEndpointParams");
-            writer.addImport("smithy_python._private", "URI");
+            writer.addImport("smithy_http.endpoints", "StaticEndpointParams");
+            writer.addImport("smithy_core", "URI");
             writer.write("""
                         # Step 7f: Invoke endpoint_resolver.resolve_endpoint
                         if config.endpoint_resolver is None:
@@ -426,8 +428,8 @@ async def _handle_attempt(
 
         writer.pushState(new SendRequestSection());
         if (context.applicationProtocol().isHttpProtocol()) {
-            writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON);
-            writer.addImport("smithy_python.interfaces.http", "HTTPRequestConfiguration");
+            writer.addDependency(SmithyPythonDependency.SMITHY_HTTP);
+            writer.addImport("smithy_http.interfaces", "HTTPRequestConfiguration");
             writer.write("""
                         # Step 7m: Invoke http_client.send
                         request_config = config.http_request_config or HTTPRequestConfiguration()

codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/ConfigGenerator.java

@@ -51,14 +51,14 @@ final class ConfigGenerator implements Runnable {
             .name("retry_strategy")
             .type(Symbol.builder()
                 .name("RetryStrategy")
-                .namespace("smithy_python.interfaces.retries", ".")
-                .addDependency(SmithyPythonDependency.SMITHY_PYTHON)
+                .namespace("smithy_core.interfaces.retries", ".")
+                .addDependency(SmithyPythonDependency.SMITHY_CORE)
                 .build())
             .documentation("The retry strategy for issuing retry tokens and computing retry delays.")
             .nullable(false)
             .initialize(writer -> {
-                writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON);
-                writer.addImport("smithy_python._private.retries", "SimpleRetryStrategy");
+                writer.addDependency(SmithyPythonDependency.SMITHY_CORE);
+                writer.addImport("smithy_core.retries", "SimpleRetryStrategy");
                 writer.write("self.retry_strategy = retry_strategy or SimpleRetryStrategy()");
             })
             .build()
@@ -71,23 +71,23 @@ final class ConfigGenerator implements Runnable {
             .name("http_client")
             .type(Symbol.builder()
                 .name("HTTPClient")
-                .namespace("smithy_python.interfaces.http", ".")
-                .addDependency(SmithyPythonDependency.SMITHY_PYTHON)
+                .namespace("smithy_http.aio.interfaces", ".")
+                .addDependency(SmithyPythonDependency.SMITHY_HTTP)
                 .build())
             .documentation("The HTTP client used to make requests.")
             .nullable(false)
             .initialize(writer -> {
-                writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON);
-                writer.addImport("smithy_python._private.http.aiohttp_client", "AIOHTTPClient");
+                writer.addDependency(SmithyPythonDependency.SMITHY_HTTP);
+                writer.addImport("smithy_http.aio.aiohttp", "AIOHTTPClient");
                 writer.write("self.http_client = http_client or AIOHTTPClient()");
             })
             .build(),
         ConfigProperty.builder()
             .name("http_request_config")
             .type(Symbol.builder()
                 .name("HTTPRequestConfiguration")
-                .namespace("smithy_python.interfaces.http", ".")
-                .addDependency(SmithyPythonDependency.SMITHY_PYTHON)
+                .namespace("smithy_http.interfaces", ".")
+                .addDependency(SmithyPythonDependency.SMITHY_HTTP)
                 .build())
             .documentation("Configuration for individual HTTP requests.")
             .build(),
@@ -102,16 +102,16 @@ final class ConfigGenerator implements Runnable {
                     .build())
                 .addReference(Symbol.builder()
                     .name("EndpointResolver")
-                    .namespace("smithy_python.interfaces.http", ".")
-                    .addDependency(SmithyPythonDependency.SMITHY_PYTHON)
+                    .namespace("smithy_http.aio.interfaces", ".")
+                    .addDependency(SmithyPythonDependency.SMITHY_HTTP)
                     .build())
                 .build())
             .documentation("""
                     The endpoint resolver used to resolve the final endpoint per-operation based on the \
                     configuration.""")
             .nullable(false)
             .initialize(writer -> {
-                writer.addImport("smithy_python._private.http", "StaticEndpointResolver");
+                writer.addImport("smithy_http.aio.endpoints", "StaticEndpointResolver");
                 writer.write("self.endpoint_resolver = endpoint_resolver or StaticEndpointResolver()");
             })
             .build(),
@@ -121,8 +121,8 @@ final class ConfigGenerator implements Runnable {
                 .name("str | URI")
                 .addReference(Symbol.builder()
                     .name("URI")
-                    .namespace("smithy_python.interfaces", ".")
-                    .addDependency(SmithyPythonDependency.SMITHY_PYTHON)
+                    .namespace("smithy_core.interfaces", ".")
+                    .addDependency(SmithyPythonDependency.SMITHY_CORE)
                     .build())
                 .build())
             .documentation("A static URI to route requests to.")
@@ -145,8 +145,8 @@ private static List<ConfigProperty> getHttpAuthProperties(GenerationContext cont
                     .name("dict[str, HTTPAuthScheme[Any, Any, Any, Any]]")
                     .addReference(Symbol.builder()
                         .name("HTTPAuthScheme")
-                        .namespace("smithy_python.interfaces.auth", ".")
-                        .addDependency(SmithyPythonDependency.SMITHY_PYTHON)
+                        .namespace("smithy_http.aio.interfaces.auth", ".")
+                        .addDependency(SmithyPythonDependency.SMITHY_HTTP)
                         .build())
                     .addReference(Symbol.builder()
                         .name("Any")
@@ -220,8 +220,8 @@ private void writeInterceptorsType(PythonWriter writer) {
                 .getContainedOperations(settings.getService());
 
         writer.addStdlibImport("typing", "Union");
-        writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON);
-        writer.getImportContainer().addImport("smithy_python.interfaces.interceptor", "Interceptor", "Interceptor");
+        writer.addDependency(SmithyPythonDependency.SMITHY_CORE);
+        writer.getImportContainer().addImport("smithy_core.interceptors", "Interceptor", "Interceptor");
 
         writer.writeInline("_ServiceInterceptor = Union[");
         var iter = operationShapes.iterator();

codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/HttpAuthGenerator.java

@@ -96,8 +96,9 @@ private void generateAuthSchemeResolver(
             .toList();
 
         writer.pushState(new GenerateHttpAuthSchemeResolverSection(resolvedAuthSchemes));
-        writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON);
-        writer.addImport("smithy_python.interfaces.auth", "HTTPAuthOption");
+        writer.addDependency(SmithyPythonDependency.SMITHY_CORE);
+        writer.addDependency(SmithyPythonDependency.SMITHY_HTTP);
+        writer.addImport("smithy_http.aio.interfaces.auth", "HTTPAuthOption");
         writer.write("""
             class $1L:
                 def resolve_auth_scheme(self, auth_parameters: $2T) -> list[HTTPAuthOption]:

codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.java

@@ -189,7 +189,7 @@ private void generateRequestTest(OperationShape operation, HttpRequestTestCase t
             } else {
                 path = "";
             }
-            writer.addImport("smithy_python._private.retries", "SimpleRetryStrategy");
+            writer.addImport("smithy_core.retries", "SimpleRetryStrategy");
             writeClientBlock(context.symbolProvider().toSymbol(service), testCase, Optional.of(() -> {
                 writer.write("""
                     config = $T(
@@ -403,8 +403,8 @@ private void writeRequestBodyComparison(HttpMessageTestCase testCase, PythonWrit
         if (testCase.getBody().isEmpty()) {
             return;
         }
-        writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON);
-        writer.addImport("smithy_python.interfaces.blobs", "AsyncBytesReader");
+        writer.addDependency(SmithyPythonDependency.SMITHY_CORE);
+        writer.addImport("smithy_core.aio.types", "AsyncBytesReader");
         writer.write("actual_body_content = await AsyncBytesReader(actual.body).read()");
         writer.write("expected_body_content = b$S", testCase.getBody().get());
         compareMediaBlob(testCase, writer);
@@ -548,9 +548,9 @@ private void assertResponseEqual(HttpMessageTestCase testCase, Shape operationOr
         for (MemberShape member : responseShape.members()) {
             var memberName = context.symbolProvider().toMemberName(member);
             if (member.equals(streamingMember)) {
-                writer.addDependency(SmithyPythonDependency.SMITHY_PYTHON);
-                writer.addImport("smithy_python.interfaces.blobs", "AsyncByteStream");
-                writer.addImport("smithy_python.interfaces.blobs", "AsyncBytesReader");
+                writer.addDependency(SmithyPythonDependency.SMITHY_CORE);
+                writer.addImport("smithy_core.aio.types", "AsyncByteStream");
+                writer.addImport("smithy_core.aio.types", "AsyncBytesReader");
                 writer.write("""
                     assert isinstance(actual.$1L, AsyncByteStream)
                     actual_body_content = await actual.$1L.read()
@@ -614,13 +614,14 @@ private void writeClientBlock(
     private void writeUtilStubs(Symbol serviceSymbol) {
         LOGGER.fine(String.format("Writing utility stubs for %s : %s", serviceSymbol.getName(), protocol.getName()));
         writer.addStdlibImport("typing", "Any");
-        writer.addImport("smithy_python.interfaces", "Fields");
-        writer.addImports("smithy_python.interfaces.http", Set.of(
-                "HTTPRequestConfiguration", "HTTPRequest", "HTTPResponse")
-        );
-        writer.addImport("smithy_python._private", "tuples_to_fields");
-        writer.addImport("smithy_python._private.http", "HTTPResponse", "_HTTPResponse");
-        writer.addImport("smithy_python.async_utils", "async_list");
+        writer.addDependency(SmithyPythonDependency.SMITHY_CORE);
+        writer.addDependency(SmithyPythonDependency.SMITHY_HTTP);
+        writer.addImport("smithy_http", "Fields");
+        writer.addImport("smithy_http.interfaces", "HTTPRequestConfiguration");
+        writer.addImports("smithy_http.aio.interfaces", Set.of("HTTPRequest", "HTTPResponse"));
+        writer.addImport("smithy_http", "tuples_to_fields");
+        writer.addImport("smithy_http.aio", "HTTPResponse", "_HTTPResponse");
+        writer.addImport("smithy_core.aio.utils", "async_list");
 
         writer.write("""
                         class $1L($2T):

codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java

@@ -29,10 +29,23 @@ public final class SmithyPythonDependency {
     /**
      * The core smithy-python python package.
      *
-     * While in development this will use the develop branch.
+     * <p>While in development this will use the develop branch.
      */
-    public static final PythonDependency SMITHY_PYTHON = new PythonDependency(
-            "smithy_python",
+    public static final PythonDependency SMITHY_CORE = new PythonDependency(
+            "smithy_core",
+            // You'll need to locally install this before we publish
+            "==0.0.1",
+            Type.DEPENDENCY,
+            false
+    );
+
+    /**
+     * The core smithy-python python package.
+     *
+     * <p>While in development this will use the develop branch.
+     */
+    public static final PythonDependency SMITHY_HTTP = new PythonDependency(
+            "smithy_http",
             // You'll need to locally install this before we publish
             "==0.0.1",
             Type.DEPENDENCY,
@@ -65,6 +78,6 @@ private SmithyPythonDependency() {}
      * @return a list of dependencies that are always needed.
      */
     public static List<SymbolDependency> getUnconditionalDependencies() {
-        return List.of(SMITHY_PYTHON.getDependency());
+        return List.of(SMITHY_CORE.getDependency());
     }
 }

codegen/smithy-python-codegen/src/main/java/software/amazon/smithy/python/codegen/SymbolVisitor.java

@@ -149,8 +149,8 @@ private String getDefaultShapeName(Shape shape) {
     public Symbol blobShape(BlobShape shape) {
         // see: https://smithy.io/2.0/spec/streaming.html#smithy-api-streaming-trait
         if (shape.hasTrait(StreamingTrait.class)) {
-            return createSymbolBuilder(shape, "StreamingBlob", "smithy_python.interfaces.blobs")
-                    .addDependency(SmithyPythonDependency.SMITHY_PYTHON)
+            return createSymbolBuilder(shape, "StreamingBlob", "smithy_core.aio.interfaces")
+                    .addDependency(SmithyPythonDependency.SMITHY_CORE)
                     .build();
         }
 
@@ -161,8 +161,8 @@ public Symbol blobShape(BlobShape shape) {
                 return createSymbolBuilder(shape, "bytes | bytearray | JsonBlob")
                         .addReference(Symbol.builder()
                                 .name("JsonBlob")
-                                .namespace("smithy_python.mediatypes", ".")
-                                .addDependency(SmithyPythonDependency.SMITHY_PYTHON)
+                                .namespace("smithy_core.types", ".")
+                                .addDependency(SmithyPythonDependency.SMITHY_CORE)
                                 .build())
                         .build();
             }
@@ -277,8 +277,8 @@ public Symbol floatShape(FloatShape shape) {
     @Override
     public Symbol documentShape(DocumentShape shape) {
         return createSymbolBuilder(shape, "Document")
-                .namespace("smithy_python.types", ".")
-                .addDependency(SmithyPythonDependency.SMITHY_PYTHON)
+                .namespace("smithy_core.types", ".")
+                .addDependency(SmithyPythonDependency.SMITHY_CORE)
                 .build();
     }
 
@@ -331,8 +331,8 @@ public Symbol stringShape(StringShape shape) {
                 return createSymbolBuilder(shape, "str | JsonString")
                         .addReference(Symbol.builder()
                                 .name("JsonString")
-                                .namespace("smithy_python.mediatypes", ".")
-                                .addDependency(SmithyPythonDependency.SMITHY_PYTHON)
+                                .namespace("smithy_core.types", ".")
+                                .addDependency(SmithyPythonDependency.SMITHY_CORE)
                                 .build())
                         .build();
             }