diff --git a/lambda-udfs/f_fastembed_embedding(varchar)/function.sql b/lambda-udfs/f_fastembed_embedding(varchar)/function.sql deleted file mode 100644 index 5d24222..0000000 --- a/lambda-udfs/f_fastembed_embedding(varchar)/function.sql +++ /dev/null @@ -1,8 +0,0 @@ -/* -Purpose: - This sample function demonstrates how use lambda to call the titan embedding model to convert your text to an embedding. - -2024-07-11: written by rjvgupta -*/ -CREATE OR REPLACE EXTERNAL FUNCTION f_fastembed_embedding (varchar) RETURNS varchar(max) STABLE -LAMBDA 'f-fastembed-embedding-varchar' IAM_ROLE ':RedshiftRole'; diff --git a/lambda-udfs/f_fastembed_embedding(varchar)/input.csv b/lambda-udfs/f_fastembed_embedding(varchar)/input.csv deleted file mode 100644 index 6cda2f2..0000000 --- a/lambda-udfs/f_fastembed_embedding(varchar)/input.csv +++ /dev/null @@ -1 +0,0 @@ -'With ComfortCut blades, you get a clean shave that''s comfortable on your skin. Rounded caps shield 27 self-sharpening blades to gently cut hair just above skin level and help the shaver glide smoothly over your skin.\n4D Flex Heads move independently in 4 directions to automatically adjust to the curves of your face, neck and jaw line.\nPop-up trimmer for mustache and sideburns Finish your look with the built-in trimmer. It’s ideal for maintaining your mustache and trimming your sideburns.\nAt the touch of a button, you can pop the heads open for an easy and thorough clean under the tap.\nUp to 40 minutes of cordless runtime - that''s about 13 shaves. Or plug it in for instant, continuous power.' diff --git a/lambda-udfs/f_fastembed_embedding(varchar)/lambda.yaml b/lambda-udfs/f_fastembed_embedding(varchar)/lambda.yaml deleted file mode 100644 index fd8387c..0000000 --- a/lambda-udfs/f_fastembed_embedding(varchar)/lambda.yaml +++ /dev/null @@ -1,91 +0,0 @@ -AWSTemplateFormatVersion: '2010-09-09' -Parameters: - S3Bucket: - Description: the S3 Bucket containing the Lambda layer - Type: String - S3Key: - Description: the S3 Key containing the Lambda layer - Type: String -Conditions: - NoS3Prefix: !Equals [!Ref S3Key, ''] -Resources: - LambdaRole: - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - - Effect: Allow - Principal: - Service: - - lambda.amazonaws.com - Action: - - sts:AssumeRole - Path: / - Policies: - - - PolicyName: CloudwatchLogs - PolicyDocument: - Version: 2012-10-17 - Statement: - - - Effect: Allow - Action: - - logs:CreateLogGroup - Resource: - - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:*" - - - Effect: Allow - Action: - - logs:CreateLogStream - - logs:PutLogEvents - Resource: - - !Sub "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/*" - FastEmbedLayer: - Type: AWS::Lambda::LayerVersion - Properties: - CompatibleRuntimes: - - python3.9 - Content: - S3Bucket: !Ref S3Bucket - S3Key: !If ["NoS3Prefix", "fastembed_0.3.3.zip", !Sub "${S3Key}/fastembed_0.3.3.zip"] - Description: FastEmbed library - LayerName: fastembed - LambdaUDFFunction: - Type: "AWS::Lambda::Function" - Properties: - FunctionName: f-fastembed-embedding-varchar - Role: !GetAtt 'LambdaRole.Arn' - Handler: index.handler - Runtime: python3.9 - Layers: - - !Ref FastEmbedLayer - Timeout: 300 - Code: - ZipFile: | - import json, sys - from fastembed import TextEmbedding - embedding_model = TextEmbedding() - - def handler(event, context): - print(event) - redshift_response = {"success": False, "num_records": event["num_records"]} - try: - result = [] - documents = [] - for row in event["arguments"]: - documents.append(row[0]) - - embeddings_list = list(embedding_model.embed(documents)) - for embedding in embeddings_list: - result.append('['+','.join(map(str, embedding))+']') - - redshift_response["success"] = True - redshift_response["results"] = result - except Exception as e: - redshift_response["error_msg"] = str(e) - print('Error: '+ str(e)) - exc_type, exc_obj, exc_tb = sys.exc_info() - print(exc_type, exc_tb.tb_lineno) - return json.dumps(redshift_response) \ No newline at end of file diff --git a/lambda-udfs/f_fastembed_embedding(varchar)/output.csv b/lambda-udfs/f_fastembed_embedding(varchar)/output.csv deleted file mode 100644 index b057f9a..0000000 --- a/lambda-udfs/f_fastembed_embedding(varchar)/output.csv +++ /dev/null @@ -1 +0,0 @@ -[-0.031330943, -0.002541473, 0.0030553224, -0.053329267, 0.0131101115, 0.08132713, 0.053773675, -0.031330943, 0.04977398, 0.027997864, 0.060884245, -0.005610683, 0.015221061, 0.024553683, 0.021776117, -0.014887753, -0.015109958, 0.0016387639, 0.014887753, -0.011610226, -0.016554292, 0.03666387, 0.06488394, 0.00358306, 0.058217783, 0.0025136971, -5.0126386e-05, -0.0012499046, 0.017554216, -0.0071105687, -0.03888592, -0.015443266, -0.015887678, 0.0069161393, 0.019220756, -0.004083022, 0.03244197, -0.024220375, 0.004305227, 0.0014998856, 0.0051940484, -0.040219154, -0.04844075, -0.007943839, -0.01766532, 0.042885616, 0.009054865, -0.020998398, -0.019665167, 0.029331096, 0.021998322, 0.013332317, 0.059995424, -0.01261015, -0.021776117, 0.0073883254, -0.049107365, -0.014665548, -0.00043052272, -0.0013193438, 0.035997253, 0.045552082, 0.00716612, 0.00883266, -0.0019720718, -0.025109196, -0.035108432, -0.04888516, -0.008554903, -0.025109196, 0.012165738, -0.0029858833, 0.012887905, -0.023220452, -0.012832355, 0.05221824, -0.021553911, 0.0059717665, -0.06843922, -0.041552387, -0.06932805, 0.017554216, 0.020553987, 0.023887066, -0.049551778, -0.0124990465, 0.020553987, -0.034664024, -0.010943609, -0.014443343, -0.012999008, -0.0046663107, 0.0506628, -0.014776651, -0.028442275, -0.045329876, -0.045774285, -0.059995424, -0.03666387, 0.013776727, -0.031108739, -0.07910508, -0.038441513, 0.04688531, 0.011332469, 0.025553606, -0.02022068, -0.007277223, -0.017776422, -0.0061661964, -0.0506628, 0.054884702, -0.0506628, -0.008666006, 0.014776651, 0.06843922, 0.021220604, -0.005610683, 0.015443266, -0.055773523, 0.02610912, -0.022998245, 0.026664633, -0.0111658145, 0.04177459, 0.035997253, 0.009110416, 0.029108891, -0.029997712, 0.0024164824, -0.038219307, -0.022998245, -0.014665548, 0.045552082, 0.025998017, -0.03533064, -0.02444258, 0.018776346, 0.028886685, 0.0017498665, 0.007499428, -0.0065550557, 0.048218545, 0.051773828, 0.020553987, 0.04932957, -0.01766532, -0.04599649, -0.008777108, 0.014221137, 0.018331936, 0.018776346, 0.024109272, -0.008277146, 0.0013540634, -0.022553835, -0.037330486, 0.013332317, -0.047774132, 0.011388021, 0.023442656, -0.01349897, 0.056440137, -0.015554369, -0.027109044, -0.078216255, -0.01855414, 0.005388478, 0.015998779, 0.013998932, -0.060884245, 0.014110034, 0.056217935, 0.011499123, -0.042885616, 0.006193972, 0.005082946, 0.02444258, -0.036886074, 0.0064161774, -0.020331783, -0.017109806, -0.0072216713, 0.0074438765, -0.050885007, -0.047107518, -0.008054941, -0.02022068, 0.02066509, -0.030219916, 0.005555132, -0.08399359, -0.05599573, -0.007666082, 0.014554446, -0.023664862, -0.023331553, -0.001458222, -0.007943839, 0.063995115, 0.07777184, 0.017776422, 0.0081104925, -0.018331936, -0.00972148, -0.01855414, -0.0019304083, 0.027553454, -0.052884854, -0.041552387, -0.03288638, -0.052884854, 0.03621946, -0.035108432, -0.024553683, -0.036886074, 0.029108891, -0.027331248, 0.021553911, 0.02566471, 0.0020970623, -0.04133018, 0.021553911, -0.02566471, -0.031330943, 0.05399588, -0.016665395, 0.041107975, -0.06577276, -0.044885464, -0.046440903, 0.0012846242, 0.01810973, -0.01349897, 0.023887066, 0.043774437, 0.014665548, -0.01933186, 0.011665776, 0.016109882, -0.025998017, 0.030442122, -0.0034580696, -0.00633285, 0.021442808, 0.033775203, -0.009388173, -0.022776041, -0.0006423121, 0.010388097, 0.005388478, -0.013610073, -0.0023887067, 0.003666387, -0.026664633, -0.007277223, -0.01944296, -0.042663414, -0.0035275086, -0.01055475, -0.016665395, 0.023442656, 0.024109272, -0.006749485, -0.06577276, 0.0631063, -0.033997405, -0.02399817, -0.010943609, -0.033775203, 0.004610759, 0.01222129, -0.03621946, -0.027109044, 0.031997558, -0.008277146, -0.006360626, -0.013665624, -0.036886074, -0.009221518, -0.04510767, -0.038441513, -0.019887371, -0.020553987, 0.04044136, 0.016332088, -0.0061661964, 0.022776041, 0.005305151, 0.05399588, 0.0035552843, -0.08221595, 0.0019442962, -0.04688531, 0.013221214, -0.0071105687, 0.02022068, -0.056662343, -0.022553835, -0.003666387, -0.044885464, 0.03621946, -0.020887295, 0.00486074, -0.0252203, 0.03244197, 0.020887295, -0.039330333, 0.008554903, -0.017887525, 0.022331629, -0.028108967, -0.0054995804, -0.044441056, -0.043552235, -0.03155315, 0.029331096, 0.026220223, -0.0033747426, -0.04088577, 0.0631063, -0.019887371, 0.048218545, -0.022664938, 0.011999085, 0.06532835, -0.013832279, 0.022331629, 0.01222129, -0.0379971, 0.009221518, -0.005832888, -0.034441818, 0.041107975, 0.012165738, 0.02444258, -0.012999008, -0.012054636, 0.02566471, 0.014554446, 0.029775506, 0.01766532, -0.03666387, 0.033552997, 0.06932805, 0.028442275, -0.054440293, 0.043996643, 0.024331477, -0.018220833, -0.062217478, 0.0049718427, 0.032219764, -0.0072216713, 0.023887066, 0.016220985, -0.0057217856, 0.006082869, -0.043552235, -0.015665472, 0.030442122, 0.051107213, -0.05399588, -0.0010971385, 0.076883025, -0.01766532, -0.015887678, 0.017554216, 0.014443343, -0.054440293, -0.021442808, -0.00633285, -0.039774742, -0.021665014, 0.027442351, -0.01944296, 0.02066509, -0.0072216713, 0.034441818, -0.04510767, 0.060884245, 0.001215185, 0.08266036, 0.0017220909, 0.009221518, -0.037774898, 0.022442732, 0.010721404, 0.03333079, 0.037552692, -0.043774437, -0.016554292, 0.027331248, -0.0023053796, 0.011221366, -0.027553454, -0.018443037, 0.021998322, 0.027220147, 0.05866219, 0.023442656, 0.0068050367, -0.031108739, 0.05777337, -0.054884702, -0.045329876, -0.005082946, 0.030886533, 0.010276994, 0.023220452, 0.043107823, 0.006610607, 0.005527356, 0.027553454, -0.022109425, 0.02566471, -0.021776117, 0.0030692103, -0.020776194, 0.0030692103, 0.0046663107, 0.0062495233, -0.029775506, -0.037330486, -0.015776575, -0.011721328, 0.021220604, 0.022109425, -0.00079507823, 0.067550406, 0.023220452, -0.018443037, -0.05599573, -2.2893219e-05, -0.010776956, -0.05266265, -0.04133018, 0.03888592, -0.017887525, 0.016332088, 0.018887449, -0.025553606, -0.030442122, 0.037774898, -0.010388097, -0.04088577, 0.0052218237, -0.03288638, -0.0042774514, -0.08043831, 0.0211095, -0.015443266, -0.022887142, 0.0027220147, 0.02566471, 0.0025553606, -0.0252203, -0.0024164824, -0.027442351, -0.05599573, 0.04044136, -0.03621946, -0.027442351, 0.014554446, 0.04977398, 0.047774132, -0.015443266, 0.04599649, 0.021665014, 0.022109425, 0.0046940865, -0.007332774, 0.020776194, -0.014998856, 0.0106658535, 0.028331172, -0.03421961, -0.040219154, -0.060884245, -0.0008645174, 0.036886074, -0.026442427, 0.0044996566, -0.0023609311, -0.024775889, 0.0081104925, 0.008443801, -0.018665243, -0.016332088, -0.0040552462, 0.029553302, 0.0077771847, -0.0211095, 0.01055475, -0.0379971, 0.023553759, 0.016109882, 0.0054995804, -0.023553759, 0.0012082412, -0.006832812, -0.013165662, -0.051107213, 0.01644319, -0.020998398, 0.016554292, 0.004638535, -0.079549484, 0.036886074, -0.027442351, -0.029775506, 0.010776956, 0.019665167, 0.018887449, -0.015332164, 0.030442122, -0.017998626, -0.0012360169, -0.0112769175, -0.047329724, -0.0021248378, 0.013165662, 0.01944296, 0.007999389, -0.0018331935, -0.03577505, -0.014776651, 0.007499428, -0.062217478, -0.040219154, -0.01349897, -0.034664024, 0.011776879, -0.013887829, -0.020442884, -0.038219307, 0.025886914, -0.019554064, 0.008221595, -0.01766532, 0.018331936, 0.012276841, 0.040219154, 0.014665548, 0.042663414, -0.014776651, -0.018887449, 0.016554292, 0.0031386495, -0.018887449, -0.007082793, -0.0073883254, 0.0087215565, -0.03710828, -0.0062495233, -0.0023470432, 0.0027497902, 0.031108739, 0.013387867, -0.032664176, 0.06355071, -0.008888211, 0.014165586, -0.039996948, -0.014221137, 0.07466097, -0.016998703, 0.028331172, 0.052884854, 0.041107975, 0.035997253, -0.008332698, 0.017998626, -0.011943533, -0.035108432, -0.03666387, 0.0027081268, -0.0022498283, 0.058217783, -0.030219916, 0.012387944, -0.016665395, 0.010388097, 0.0009374285, -0.0013123999, -0.011943533, -0.02655353, -0.038441513, -0.051773828, 0.0023887067, -0.0036941627, 0.016776498, -0.039330333, 0.008166044, -0.0011249142, 0.0019026326, -0.008388249, 0.027109044, 0.034441818, 0.031330943, -0.00716612, -0.005277375, -0.0028331173, -0.0026664634, -0.042663414, 0.006749485, -0.023664862, 0.014443343, 0.027997864, -0.07332774, 0.013443419, -0.03666387, 0.04688531, 0.00972148, -0.012110188, 0.0028886686, -0.003916368, 0.0010346433, 0.00015450211, 0.04421885, -0.031997558, -0.025886914, -0.037330486, -0.054662496, 0.020331783, -0.0011874094, 0.008277146, -0.041107975, 0.005860664, -0.006777261, -0.048662953, -0.044441056, 0.021887219, 0.033108585, 0.03533064, -0.01011034, -0.024331477, -0.0012429607, 0.06799481, 0.02699794, -0.013276765, -0.00883266, 0.00095826027, 0.018776346, -0.05732896, -0.012721252, 0.0018609691, -0.01944296, 0.04688531, 0.029997712, 0.031108739, 0.020331783, 0.01810973, 0.05155162, -0.031330943, -0.0034302939, 0.103547655, 0.015665472, 0.014054484, 0.059995424, -0.02699794, 0.0029719956, 0.011665776, 0.0033330792, 0.015109958, -0.0008332698, 0.034886226, -0.023553759, 0.011665776, 0.0022359407, -0.049996186, 0.014887753, 0.04888516, -0.028886685, -0.010610302, 0.0068883635, 0.021665014, -0.034441818, 0.023664862, 0.01766532, -0.0087215565, 0.05866219, 0.026886838, 0.014054484, -0.029331096, 0.047107518, 0.0131101115, -0.0044718813, 0.023775965, 0.03155315, 0.079993896, -0.020553987, 0.017998626, -0.00927707, 0.02022068, -0.006499504, -0.031330943, -0.008554903, 0.029553302, -0.034886226, 0.016332088, 0.0029025564, 0.036886074, -0.018665243, 0.011443571, -0.0054718046, -0.027109044, 0.0056662345, 0.035997253, 0.037552692, -0.033552997, -0.021442808, -0.014165586, 0.05155162, 0.011665776, 0.0023748188, 0.017887525, 0.0028886686, 0.015443266, 0.024331477, -0.00095826027, 0.0073883254, 0.00045829837, 0.036441665, -0.0004478825, -0.0054995804, -0.020331783, 0.055773523, -0.013554522, 0.012332393, -0.011499123, 0.010610302, -0.021998322, -0.05155162, -0.02488699, -0.01305456, 0.020776194, -0.00022741321, 0.026775734, 0.017776422, -0.017220909, 0.050218392, 0.01055475, 0.023220452, 0.07421656, -0.026442427, 0.015887678, 0.030886533, -0.01644319, -0.09777032, -0.0027497902, 0.0073883254, 0.007943839, -0.024664786, 0.0631063, -0.0093326215, 0.052440446, -0.014776651, -0.0056384588, -0.049551778, 0.10488089, 0.04044136, 0.061328657, 0.015332164, -0.055329114, -0.03244197, -0.033775203, 0.022887142, 0.014554446, -0.050218392, 0.012387944, 0.038219307, 0.04799634, 0.017109806, 0.0047774133, 0.014887753, 0.037552692, 0.0011318581, 0.021331707, 0.07777184, -0.021220604, 0.011610226, 0.022220528, 0.011554674, -0.059106603, 0.033775203, -0.045774285, 0.040663566, -0.039108127, 0.008499351, 0.0006075925, -0.022887142, -0.063995115, -0.011332469, 0.01766532, 0.014776651, 0.007943839, -0.0035413965, -0.04421885, -0.041996796, -0.0017290348, 0.016665395, -0.010054789, -0.032664176, 0.027109044, -0.014998856, -0.0054995804, -0.037774898, 0.06932805, 0.0253314, 0.00179153, -0.0252203, 0.05355147, 0.012443495, 0.0504406, -0.0030414346, 0.018665243, 0.015998779, -0.01977627, -0.0010207555, 0.059995424, 0.0035413965, -0.033108585, -0.010221442, -0.01977627, 0.0065550557, -0.033552997, -0.026220223, 0.029775506, -0.05688455, 0.031330943, 0.0059162155, 0.068883635, -0.039108127, 0.07288333, 0.027886761, -0.034886226, -0.036441665, -0.00035413966, 0.017220909, 0.01305456, 0.022331629, -0.010832507, -0.010943609, 0.001874857, -0.02444258, -0.017887525, -0.015887678, 0.00966593, 0.023442656, -0.029775506, -0.02566471, 0.009388173, -0.043996643, 0.0046663107, 0.021331707, 0.07110569, 0.040219154, 0.05399588, 0.06488394, -0.018220833, 0.051996034, 0.05355147, -0.035552844, 0.0035413965, 0.011110264, -0.004805189, -0.031997558, -0.0018609691, -0.011610226, 0.014665548, -0.01644319, 0.030442122, -0.022109425, 0.00051732163, -0.02866448, 0.0031108737, -0.01766532, -0.011554674, -0.01933186, -0.0066939336, -0.050885007, 0.010221442, -0.005082946, -0.00441633, 0.017109806, 0.00018140978, 0.032664176, 0.0026109118, -0.06621717, -0.039996948, 0.00042184282, -0.018887449, -0.024775889, -0.023220452, -0.0045274324, -0.022776041, 0.01261015, -0.011443571, -0.033997405, -0.011499123, -0.011054712, -0.040219154, 0.014887753, 0.00010112076, 0.0061661964, -0.00716612, -0.049996186, -0.045774285, -0.01899855, -0.010832507, -0.0020831744, -0.055106908, 0.0011596337, -0.0112769175, -0.0039441437, 0.019887371, -0.012832355, -0.033775203, 0.0093326215, -0.028442275, 0.031775355, -0.026331324, -0.008221595, -0.029108891, -0.023442656, 0.0017776422, -0.044441056, 0.010388097, -0.020887295, 0.013443419, -0.024553683, 0.0081104925, -0.056440137, 0.00063884014, -0.0074438765, 0.022998245, -0.016332088, -0.00019876956, -0.025442503, -0.024109272, 0.062661886, 0.023887066, -0.03533064, 0.020887295, 0.053773675, -0.02610912, -0.061328657, 0.026886838, -0.008943763, -0.024775889, -0.028886685, -0.005610683, 0.038219307, 0.0506628, 0.034441818, -0.08799329, 0.016665395, 0.0106658535, 0.0041941246, 0.0034580696, 0.0029025564, 0.024331477, 0.05599573, -0.013332317, 0.0074438765, -0.062217478, 0.043107823, 0.030219916, -0.059995424, -0.00065272796, -0.005555132, 0.01855414, 0.007082793, 0.008943763, 0.028442275, -0.06577276, 0.0759942, -0.011665776, -0.026442427, 0.004888516, -0.01305456, -0.0253314, 0.02022068, 0.032664176, -0.039108127, -0.016776498, 0.0063884016, -0.027109044, 0.024331477, -0.054440293, -0.033775203, -0.034664024, -0.027220147, -0.00043746663, -0.030664328, 0.010332545, 0.032664176, -0.012776803, 0.06621717, -0.013387867, 0.14843312, 0.029108891, -0.011443571, 0.026664633, 0.013998932, 0.032219764, 0.0253314, -0.0069161393, -0.02577581, -0.062217478, -0.012554598, 0.0087215565, 0.0052495995, 0.05866219, -0.014443343, 0.055106908, 0.0111658145, -0.030886533, -0.049107365, -0.026775734, -0.038441513, 0.0081104925, -0.0040274705] \ No newline at end of file diff --git a/lambda-udfs/f_fastembed_embedding(varchar)/requirements.txt b/lambda-udfs/f_fastembed_embedding(varchar)/requirements.txt deleted file mode 100644 index f09cb8b..0000000 --- a/lambda-udfs/f_fastembed_embedding(varchar)/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -fastembed==0.3.3 \ No newline at end of file