Adding a Json provider in the same vein as the Xml provider.

src/Paillave.Etl.Json/Core/IJsonNodeDefinition.cs

@@ -0,0 +1,12 @@
+using Paillave.Etl.Json.Core.Mapping;
+
+namespace Paillave.Etl.Json.Core
+{
+    public interface IJsonNodeDefinition
+    {
+        string Name { get; }
+        string NodePath { get; }
+        Type Type { get; }
+        IList<JsonFieldDefinition> GetJsonFieldDefinitions();
+    }
+}

src/Paillave.Etl.Json/Core/JsonFileDefinition.cs

@@ -0,0 +1,35 @@
+using System.Linq.Expressions;
+using Paillave.Etl.Json.Core.Mapping;
+
+namespace Paillave.Etl.Json.Core
+{
+    public class JsonFileDefinition
+    {
+        public Dictionary<string, string> PrefixToUriNameSpacesDictionary { get; } = new Dictionary<string, string>();
+        internal List<IJsonNodeDefinition> JsonNodeDefinitions { get; } = new List<IJsonNodeDefinition>();
+
+        public JsonFileDefinition AddNameSpace(string prefix, string uri)
+        {
+            this.PrefixToUriNameSpacesDictionary[prefix] = uri;
+            return this;
+        }
+
+        public JsonFileDefinition AddNameSpaces(IDictionary<string, string> _prefixToUriNameSpacesDictionary)
+        {
+            foreach (var item in _prefixToUriNameSpacesDictionary)
+                this.PrefixToUriNameSpacesDictionary[item.Key] = item.Value;
+            return this;
+        }
+
+        public JsonFileDefinition AddNodeDefinition(IJsonNodeDefinition xmlNodeDefinition)
+        {
+            this.JsonNodeDefinitions.Add(xmlNodeDefinition);
+            return this;
+        }
+        public JsonFileDefinition AddNodeDefinition<T>(string name, string nodeXPath, Expression<Func<IJsonFieldMapper, T>> expression)
+        {
+            this.JsonNodeDefinitions.Add(JsonNodeDefinition.Create<T>(name, nodeXPath, expression));
+            return this;
+        }
+    }
+}

src/Paillave.Etl.Json/Core/JsonNodeDefinition.cs

@@ -0,0 +1,44 @@
+using Paillave.Etl.Json.Core.Mapping;
+using Paillave.Etl.Json.Core.Mapping.Visitors;
+using System.Linq.Expressions;
+
+namespace Paillave.Etl.Json.Core
+{
+    public static class JsonNodeDefinition
+    {
+        public static JsonNodeDefinition<T> Create<T>(string name, string nodePath, Expression<Func<IJsonFieldMapper, T>> expression)
+            => new JsonNodeDefinition<T>(name, nodePath).WithMap(expression);
+    }
+    public class JsonNodeDefinition<T> : IJsonNodeDefinition
+    {
+        public string Name { get; set; }
+        public IList<JsonFieldDefinition> _jsonFieldDefinitions = new List<JsonFieldDefinition>();
+
+        public IList<JsonFieldDefinition> GetJsonFieldDefinitions() => _jsonFieldDefinitions.ToList();
+        public string NodePath { get; private set; }
+
+        public Type Type { get; } = typeof(T);
+
+        public JsonNodeDefinition(string name, string nodePath)
+        {
+            this.Name = name;
+            this.NodePath = nodePath;
+        }
+        public JsonNodeDefinition<T> WithMap(Expression<Func<IJsonFieldMapper, T>> expression)
+        {
+            JsonMapperVisitor vis = new JsonMapperVisitor();
+            vis.Visit(expression);
+            foreach (var item in vis.MappingSetters)
+                this.SetFieldDefinition(item);
+            return this;
+        }
+        private void SetFieldDefinition(JsonFieldDefinition jsonFieldDefinition)
+        {
+            var existingFieldDefinition = _jsonFieldDefinitions.FirstOrDefault(i => i.TargetPropertyInfo.Name == jsonFieldDefinition.TargetPropertyInfo.Name);
+            if (existingFieldDefinition == null)
+                _jsonFieldDefinitions.Add(jsonFieldDefinition);
+            else
+                if (jsonFieldDefinition.NodePath != null) existingFieldDefinition.NodePath = jsonFieldDefinition.NodePath;
+        }
+    }
+}

src/Paillave.Etl.Json/Core/JsonNodeParsed.cs

@@ -0,0 +1,15 @@
+namespace Paillave.Etl.Json.Core
+{
+    public class JsonNodeParsed
+    {
+        public string SourceName { get; internal set; }
+        public string NodeDefinitionName { get; internal set; }
+        public string NodePath { get; internal set; }
+        public Type Type { get; internal set; }
+        public object Value { get; internal set; }
+        public T GetValue<T>() => (T)Value;
+        // public object[] ParentValues { get; internal set; }
+        // public T GetValue<T>(int level = 0) => (T)(level == 0 ? Value : ParentValues[level - 1]);
+        public HashSet<Guid> CorrelationKeys { get; set; } = new HashSet<Guid>();
+    }
+}

src/Paillave.Etl.Json/Core/JsonObjectReader.cs

@@ -0,0 +1,177 @@
+using Paillave.Etl.Core;
+using Paillave.Etl.Json.Core.Mapping;
+using System.Buffers;
+using System.Runtime.Serialization.Json;
+using System.Text.Json;
+using System.Xml;
+
+namespace Paillave.Etl.Json.Core
+{
+    public class JsonObjectReader
+    {
+        private class JsonReadField
+        {
+            public JsonFieldDefinition Definition { get; set; }
+            public IJsonNodeDefinition NodeDefinition { get; set; }
+            public int Depth { get; set; }
+            public object Value { get; set; }
+        }
+
+        private HashSet<string> _jsonFieldsDefinitionSearch;
+        private HashSet<string> _jsonNodesDefinitionSearch;
+
+        private readonly List<JsonReadField> _inScopeReadFields = new List<JsonReadField>();
+        private readonly JsonFileDefinition _jsonFileDefinition;
+
+        public JsonObjectReader(JsonFileDefinition jsonFileDefinition)
+        {
+            _jsonFileDefinition = jsonFileDefinition;
+            _jsonNodesDefinitionSearch = new HashSet<string>(jsonFileDefinition.JsonNodeDefinitions.Select(i => i.NodePath).Distinct());
+            _jsonFieldsDefinitionSearch = new HashSet<string>(jsonFileDefinition.JsonNodeDefinitions.SelectMany(nd => nd.GetJsonFieldDefinitions().Select(fd => fd.NodePath)).Distinct());
+        }
+        private bool JsonReadFieldShouldBeCleanedUp(JsonReadField jsonReadField, int depth)
+        {
+            var depthScope = jsonReadField.Definition.DepthScope;
+            int depthLimit;
+            if (depthScope > 0)
+                depthLimit = depthScope;
+            else
+                depthLimit = jsonReadField.Depth + depthScope;
+            return depth < depthLimit;
+        }
+        private void ProcessEndOfAnyNode(Stack<NodeLevel> nodes)
+        {
+            foreach (var item in _inScopeReadFields.Where(i => JsonReadFieldShouldBeCleanedUp(i, nodes.Count - 1)).ToList())
+                _inScopeReadFields.Remove(item);
+        }
+        private void ProcessAttributeValue(string key, Stack<NodeLevel> nodes, string stringContent)
+        {
+            // string key = $"/{string.Join("/", nodes.Reverse())}";
+            if (!_jsonFieldsDefinitionSearch.Contains(key)) return;
+            var fds = _jsonFileDefinition.JsonNodeDefinitions.SelectMany(nd => nd.GetJsonFieldDefinitions().Select(fd => new { Fd = fd, Nd = nd })).Where(i => i.Fd.NodePath == key).ToList();
+            if (string.IsNullOrWhiteSpace(stringContent))
+            {
+                foreach (var fd in fds)
+                {
+                    _inScopeReadFields.Add(new JsonReadField
+                    {
+                        Depth = nodes.Count - 1,
+                        Definition = fd.Fd,
+                        NodeDefinition = fd.Nd,
+                        Value = null
+                    });
+                }
+            }
+            else
+            {
+                foreach (var fd in fds)
+                {
+                    _inScopeReadFields.Add(new JsonReadField
+                    {
+                        Depth = nodes.Count - 1,
+                        Definition = fd.Fd,
+                        NodeDefinition = fd.Nd,
+                        Value = fd.Fd.Convert(stringContent)
+                    });
+                }
+            }
+        }
+        private string ComputeKey(Stack<NodeLevel> nodes) => $"/{string.Join("/", nodes.Select(i => i.Name).Reverse())}";
+        private void ProcessEndOfNode(Stack<NodeLevel> nodes, string text, Action<JsonNodeParsed> pushResult, string sourceName)
+        {
+            string key = ComputeKey(nodes);
+            if (_jsonFieldsDefinitionSearch.Contains(key))
+            {
+                ProcessAttributeValue(key, nodes, text);
+            }
+            else if (_jsonNodesDefinitionSearch.Contains(key))
+            {
+                var (value, nd) = CreateValue(sourceName, key);
+                pushResult(new JsonNodeParsed
+                {
+                    NodeDefinitionName = nd.Name,
+                    SourceName = sourceName,
+                    NodePath = nd.NodePath,
+                    Type = nd.Type,
+                    Value = value,
+                    CorrelationKeys = nodes.Select(i => i.Guid).Where(i => i.HasValue).Select(i => i.Value).ToHashSet()
+                });
+            }
+            ProcessEndOfAnyNode(nodes);
+        }
+
+        private (object value, IJsonNodeDefinition nd) CreateValue(string sourceName, string key)
+        {
+            var nd = _jsonFileDefinition.JsonNodeDefinitions.FirstOrDefault(i => i.NodePath == key);
+            var objectBuilder = new ObjectBuilder(nd.Type);
+            foreach (var inScopeReadField in _inScopeReadFields.Where(rf => rf.NodeDefinition.NodePath == key))
+                objectBuilder.Values[inScopeReadField.Definition.TargetPropertyInfo.Name] = inScopeReadField.Value;
+            foreach (var propName in nd.GetJsonFieldDefinitions().Where(i => i.ForRowGuid).Select(i => i.TargetPropertyInfo.Name).ToList())
+                objectBuilder.Values[propName] = Guid.NewGuid();
+            foreach (var propName in nd.GetJsonFieldDefinitions().Where(i => i.ForSourceName).Select(i => i.TargetPropertyInfo.Name).ToList())
+                objectBuilder.Values[propName] = sourceName;
+            return (objectBuilder.CreateInstance(), nd);
+        }
+
+        public void Read(Stream fileStream, string sourceName, Action<JsonNodeParsed> pushResult, CancellationToken cancellationToken)
+        {
+            var readerSettings = new JsonReaderOptions();
+            readerSettings.CommentHandling = JsonCommentHandling.Skip;
+
+            using var streamReader = new StreamReader(fileStream);
+            var jsonReader = new Utf8JsonReader(new ReadOnlySequence<byte>(streamReader.ReadToEnd().Select(Convert.ToByte).ToArray()), readerSettings);
+
+            var nodes = new Stack<NodeLevel>();
+            string lastPropertyName = null;
+            string lastArrayPropertyName = null;
+            while (jsonReader.Read())
+            {
+                if (cancellationToken.IsCancellationRequested) break;
+                switch (jsonReader.TokenType)
+                {
+                    case JsonTokenType.PropertyName:
+                        lastPropertyName = jsonReader.GetString();
+                        break;
+
+                    case JsonTokenType.EndObject:
+                        if (nodes.Any())
+                        { 
+                            ProcessEndOfNode(nodes, null, pushResult, sourceName);
+                            nodes.Pop();
+                        }
+                        break;
+
+                    case JsonTokenType.StartArray:
+                        lastArrayPropertyName = lastPropertyName;
+                        break;
+
+                    case JsonTokenType.EndArray:
+                        lastArrayPropertyName = null;
+                        break;
+
+                    case JsonTokenType.StartObject:
+                        if (!string.IsNullOrEmpty(lastPropertyName))
+                        {
+                            nodes.Push(new NodeLevel { Name = lastArrayPropertyName ?? lastPropertyName, Guid = Guid.NewGuid() });
+                        }
+                        break;
+
+                    // read values into last-value var
+                    case JsonTokenType.String:
+                    case JsonTokenType.Number:
+                    case JsonTokenType.True:
+                    case JsonTokenType.False:
+                        nodes.Push(new NodeLevel { Name = lastPropertyName, Guid = Guid.NewGuid() });
+                        ProcessEndOfNode(nodes, jsonReader.GetString(), pushResult, sourceName);
+                        nodes.Pop();
+                        break;
+                }
+            }
+        }
+        private struct NodeLevel
+        {
+            public string Name { get; set; }
+            public Guid? Guid { get; set; }
+        }
+    }
+}

src/Paillave.Etl.Json/Core/Mapping/IJsonFieldMapper.cs

@@ -0,0 +1,10 @@
+namespace Paillave.Etl.Json.Core.Mapping
+{
+    public interface IJsonFieldMapper
+    {
+        T ToPathQuery<T>(string xPathQuery);
+        T ToPathQuery<T>(string xPathQuery, int depthScope);
+        string ToSourceName();
+        Guid ToRowGuid();
+    }
+}

src/Paillave.Etl.Json/Core/Mapping/JsonFieldDefinition.cs

@@ -0,0 +1,46 @@
+using System.Reflection;
+using System.Xml;
+
+namespace Paillave.Etl.Json.Core.Mapping
+{
+    public class JsonFieldDefinition
+    {
+        static JsonFieldDefinition()
+        {
+            _typeConverters = typeof(XmlConvert)
+                .GetMethods(BindingFlags.Static | BindingFlags.Public)
+                .Where(i => i.Name.StartsWith("To") && i.GetParameters().Count() == 1 && i.GetParameters()[0].ParameterType == typeof(string))
+                .ToDictionary(i => i.ReturnType);
+        }
+        private MethodInfo _convertMethod = null;
+        private static readonly Dictionary<Type, MethodInfo> _typeConverters;
+        public int DepthScope { get; internal set; } = 0;
+        public string NodePath { get; internal set; } = null;
+        public bool ForSourceName { get; internal set; } = false;
+        public bool ForRowGuid { get; internal set; } = false;
+        private PropertyInfo _targetPropertyInfo = null;
+        public PropertyInfo TargetPropertyInfo
+        {
+            get => _targetPropertyInfo;
+            internal set
+            {
+                _targetPropertyInfo = value;
+                var underlyingType = Nullable.GetUnderlyingType(_targetPropertyInfo.PropertyType);
+                IsNullableProperty = underlyingType != null;
+                if (!IsNullableProperty)
+                    underlyingType = _targetPropertyInfo.PropertyType;
+                if (underlyingType == typeof(string))
+                    _convertMethod = null;
+                else
+                    _convertMethod = _typeConverters[underlyingType];
+            }
+        }
+        public bool IsNullableProperty { get; private set; }
+        public object Convert(string stringValue)
+        {
+            if (_convertMethod != null)
+                return _convertMethod.Invoke(null, new object[] { stringValue });
+            else return stringValue;
+        }
+    }
+}

src/Paillave.Etl.Json/Core/Mapping/JsonMappingProcessor.cs

@@ -0,0 +1,17 @@
+using Paillave.Etl.Json.Core.Mapping.Visitors;
+using System.Linq.Expressions;
+
+namespace Paillave.Etl.Json.Core.Mapping
+{
+    public class JsonMappingProcessor<T>
+    {
+        public readonly List<JsonFieldDefinition> _mappingSetters;
+
+        public JsonMappingProcessor(Expression<Func<IJsonFieldMapper, T>> expression)
+        {
+            JsonMapperVisitor vis = new JsonMapperVisitor();
+            vis.Visit(expression);
+            this._mappingSetters = vis.MappingSetters;
+        }
+    }
+}

src/Paillave.Etl.Json/Core/Mapping/Visitors/DummyFieldMapper.cs

@@ -0,0 +1,34 @@
+using System;
+
+namespace Paillave.Etl.Json.Core.Mapping.Visitors
+{
+    public class DummyFieldMapper : IJsonFieldMapper
+    {
+        public JsonFieldDefinition MappingSetter { get; } = new JsonFieldDefinition();
+
+        public string ToSourceName()
+        {
+            this.MappingSetter.ForSourceName = true;
+            return default;
+        }
+
+        public Guid ToRowGuid()
+        {
+            this.MappingSetter.ForRowGuid = true;
+            return default;
+        }
+
+        public T ToPathQuery<T>(string xPathQuery)
+        {
+            this.MappingSetter.NodePath = xPathQuery;
+            return default;
+        }
+
+        public T ToPathQuery<T>(string xPathQuery, int depthScope)
+        {
+            this.MappingSetter.NodePath = xPathQuery;
+            this.MappingSetter.DepthScope = depthScope;
+            return default;
+        }
+    }
+}