Add possibility to embed custom libraries into the sketch

docs/sketch-build-process.md

@@ -52,19 +52,19 @@ generated for the path to each library dependency and appended to the
 If multiple libraries contain a file that matches the `#include` directive, the priority is determined by applying the
 following rules, one by one in this order, until a rule determines a winner:
 
-1. A library that has been specified using the [`--library` option](commands/arduino-cli_compile.md#options) of
-   `arduino-cli compile` wins against a library in other locations
+1. A library has been specified using the [`--library` option](commands/arduino-cli_compile.md#options) of
+   `arduino-cli compile`.
+1. A library is found in the `libraries` subfolder of the sketch.
 1. A library that is architecture compatible wins against a library that is not architecture compatible (see
    [**Architecture Matching**](#architecture-matching))
 1. A library with both [library name](#library-name-priority) and [folder name](#folder-name-priority) matching the
-   include wins
-1. A library that has better "library name priority" or "folder name priority" wins (see
-   [**Library Name Priority**](#library-name-priority) and [**Folder Name Priority**](#folder-name-priority))
-1. A library that is architecture optimized wins against a library that is not architecture optimized (see
-   [**Architecture Matching**](#architecture-matching))
-1. A library that has a better "location priority" wins (see [**Location Priority**](#location-priority))
-1. A library that has a folder name with a better score using the "closest-match" algorithm wins
-1. A library that has a folder name that comes first in alphanumeric order wins
+   include.
+1. A library has better "library name priority" or "folder name priority" (see
+   [**Library Name Priority**](#library-name-priority) and [**Folder Name Priority**](#folder-name-priority)).
+1. A library is architecture optimized (see [**Architecture Matching**](#architecture-matching)).
+1. A library has a better "location priority" (see [**Location Priority**](#location-priority)).
+1. A library has a folder name with a better score using the "closest-match" algorithm.
+1. A library has a folder name that comes first in alphanumeric order.
 
 ### Architecture Matching
 
@@ -126,14 +126,15 @@ The "location priority" is determined as follows (in order of highest to lowest
 
 1. The library is under a custom libraries path specified via the
    [`--libraries` option](commands/arduino-cli_compile.md#options) of `arduino-cli compile` (in decreasing order of
-   priority when multiple custom paths are defined)
-1. The library is under the `libraries` subfolder of the IDE's sketchbook or Arduino CLI's user directory
+   priority when multiple custom paths are defined).
+1. The library is under the `libraries` subfolder of the sketch.
+1. The library is under the `libraries` subfolder of the IDE's sketchbook or Arduino CLI's user directory.
 1. The library is bundled with the board platform/core
-   ([`{runtime.platform.path}/libraries`](platform-specification.md#global-predefined-properties))
+   ([`{runtime.platform.path}/libraries`](platform-specification.md#global-predefined-properties)).
 1. The library is bundled with the [referenced](platform-specification.md#referencing-another-core-variant-or-tool)
-   board platform/core
+   board platform/core.
 1. The library is bundled with the Arduino IDE (this location is determined by the Arduino CLI configuration setting
-   `directories.builtin.libraries`)
+   `directories.builtin.libraries`).
 
 #### Location priorities in Arduino Web Editor
 

docs/sketch-specification.md

@@ -72,6 +72,14 @@ Files added to the sketch via the Arduino IDE's **Sketch > Add File...** are pla
 The Arduino IDE's **File > Save As...** only copies the code files in the sketch root folder and the full contents of
 the `data` folder, so any non-code files outside the `data` folder are stripped.
 
+### `libraries` subfolder
+
+The `libraries` folder is used to store libraries compiled with the sketch. This folder should be used to store
+libraries that have been patched or to store libraries that are not available through the official library repository.
+
+- This feature is available since Arduino CLI 1.1.1
+- This feature is not yet available in Arduino IDE and Arduino Web Editor.
+
 ### Metadata
 
 #### `sketch.json`
@@ -118,25 +126,27 @@ Web Editor.
 ### Sketch file structure example
 
 ```
-Foo
+MotorController
 |_ arduino_secrets.h
-|_ Abc.ino
-|_ Def.cpp
-|_ Def.h
-|_ Foo.ino
-|_ Ghi.c
-|_ Ghi.h
-|_ Jkl.h
-|_ Jkl.S
+|_ motors.ino
+|_ defs.cpp
+|_ defs.h
+|_ MotorController.ino
+|_ someASM.h
+|_ someASM.S
 |_ sketch.yaml
 |_ data
 |  |_ Schematic.pdf
+|_ libraries
+|  |_ SomeLib
+|     |_ library.properties
+|     |_ src
+|        |_ SomeLib.h
+|        |_ SomeLib.cpp
 |_ src
-   |_ SomeLib
-      |_ library.properties
-      |_ src
-         |_ SomeLib.h
-         |_ SomeLib.cpp
+   |_ encoders
+      |_ encoders.h
+      |_ encoders.cpp
 ```
 
 ## Sketchbook

internal/arduino/builder/builder.go

@@ -191,7 +191,7 @@ func NewBuilder(
 	logger := logger.New(stdout, stderr, verbose, warningsLevel)
 	libsManager, libsResolver, verboseOut, err := detector.LibrariesLoader(
 		useCachedLibrariesResolution, librariesManager,
-		builtInLibrariesDirs, libraryDirs, otherLibrariesDirs,
+		sk, builtInLibrariesDirs, libraryDirs, otherLibrariesDirs,
 		actualPlatform, targetPlatform,
 	)
 	if err != nil {

internal/arduino/builder/internal/detector/detector.go

@@ -597,6 +597,7 @@ func (f *sourceFile) DepfilePath() *paths.Path {
 func LibrariesLoader(
 	useCachedLibrariesResolution bool,
 	librariesManager *librariesmanager.LibrariesManager,
+	sk *sketch.Sketch,
 	builtInLibrariesDirs *paths.Path, libraryDirs, otherLibrariesDirs paths.PathList,
 	actualPlatform, targetPlatform *cores.PlatformRelease,
 ) (*librariesmanager.LibrariesManager, *librariesresolver.Cpp, []byte, error) {
@@ -667,7 +668,7 @@ func LibrariesLoader(
 	}
 
 	allLibs := lm.FindAllInstalled()
-	resolver := librariesresolver.NewCppResolver(allLibs, targetPlatform, actualPlatform)
+	resolver := librariesresolver.NewCppResolver(allLibs, sk, targetPlatform, actualPlatform)
 	return lm, resolver, verboseOut.Bytes(), nil
 }
 

internal/arduino/libraries/libraries_location.go

@@ -40,6 +40,8 @@ const (
 	// Unmanaged is for libraries set manually by the user in the CLI command or from the gRPC function.
 	// Ideally it's used for `libraries` outside folders managed by the CLI.
 	Unmanaged
+	// Sketch is for libraries that are part of the sketch (inside the `libraries` subfolder of the sketch).
+	Sketch
 )
 
 func (d *LibraryLocation) String() string {
@@ -54,6 +56,8 @@ func (d *LibraryLocation) String() string {
 		return "user"
 	case Unmanaged:
 		return "unmanaged"
+	case Sketch:
+		return "sketch"
 	default:
 		panic(fmt.Sprintf("invalid LibraryLocation value %d", *d))
 	}
@@ -86,6 +90,9 @@ func (d *LibraryLocation) UnmarshalJSON(b []byte) error {
 	case "unmanaged":
 		*d = Unmanaged
 		return nil
+	case "sketch":
+		*d = Sketch
+		return nil
 	default:
 		return errors.New(i18n.Tr("invalid library location: %s", s))
 	}
@@ -104,6 +111,8 @@ func (d *LibraryLocation) ToRPCLibraryLocation() rpc.LibraryLocation {
 		return rpc.LibraryLocation_LIBRARY_LOCATION_USER
 	case Unmanaged:
 		return rpc.LibraryLocation_LIBRARY_LOCATION_UNMANAGED
+	case Sketch:
+		return rpc.LibraryLocation_LIBRARY_LOCATION_SKETCH
 	default:
 		panic(fmt.Sprintf("invalid LibraryLocation value %d", *d))
 	}
@@ -122,6 +131,8 @@ func FromRPCLibraryLocation(l rpc.LibraryLocation) LibraryLocation {
 		return User
 	case rpc.LibraryLocation_LIBRARY_LOCATION_UNMANAGED:
 		return Unmanaged
+	case rpc.LibraryLocation_LIBRARY_LOCATION_SKETCH:
+		return Sketch
 	default:
 		panic(fmt.Sprintf("invalid rpc.LibraryLocation value %d", l))
 	}

internal/arduino/libraries/libraries_test.go

@@ -50,6 +50,7 @@ func TestLibLayoutAndLocationJSONUnMarshaler(t *testing.T) {
 	testLocation(ReferencedPlatformBuiltIn)
 	testLocation(User)
 	testLocation(Unmanaged)
+	testLocation(Sketch)
 }
 
 func TestLibrariesLoader(t *testing.T) {

internal/arduino/libraries/librariesresolver/cpp.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/arduino/arduino-cli/internal/arduino/cores"
 	"github.com/arduino/arduino-cli/internal/arduino/libraries"
+	"github.com/arduino/arduino-cli/internal/arduino/sketch"
 	"github.com/arduino/arduino-cli/internal/arduino/utils"
 	"github.com/arduino/arduino-cli/internal/i18n"
 	"github.com/schollz/closestmatch"
@@ -35,7 +36,7 @@ type Cpp struct {
 }
 
 // NewCppResolver creates a new Cpp resolver
-func NewCppResolver(allLibs []*libraries.Library, targetPlatform, actualPlatform *cores.PlatformRelease) *Cpp {
+func NewCppResolver(allLibs []*libraries.Library, sk *sketch.Sketch, targetPlatform, actualPlatform *cores.PlatformRelease) *Cpp {
 	resolver := &Cpp{
 		headers: map[string]libraries.List{},
 	}
@@ -45,10 +46,17 @@ func NewCppResolver(allLibs []*libraries.Library, targetPlatform, actualPlatform
 	if actualPlatform != targetPlatform {
 		resolver.ScanPlatformLibraries(allLibs, actualPlatform)
 	}
-
+	resolver.ScanSketchLibraries(sk)
 	return resolver
 }
 
+// ScanSketchLibraries loads libraries bundled with the sketch
+func (resolver *Cpp) ScanSketchLibraries(sk *sketch.Sketch) {
+	for _, lib := range sk.VendoredLibraries() {
+		_ = resolver.ScanLibrary(lib)
+	}
+}
+
 // ScanIDEBuiltinLibraries reads ide-builtin librariers loaded in the LibrariesManager to find
 // and cache all C++ headers for later retrieval.
 func (resolver *Cpp) ScanIDEBuiltinLibraries(allLibs []*libraries.Library) {
@@ -199,9 +207,12 @@ func ComputePriority(lib *libraries.Library, header, arch string) int {
 		priority += 2
 	case libraries.User:
 		priority += 3
+	case libraries.Sketch:
+		// Bonus for sketch libraries, those libraries get a better priority than others
+		priority += 10000
 	case libraries.Unmanaged:
 		// Bonus for libraries specified via --libraries flags, those libraries gets the highest priority
-		priority += 10000
+		priority += 20000
 	default:
 		panic(fmt.Sprintf("Invalid library location: %d", lib.Location))
 	}

internal/arduino/sketch/sketch.go

@@ -26,20 +26,22 @@ import (
 	"github.com/arduino/arduino-cli/commands/cmderrors"
 	f "github.com/arduino/arduino-cli/internal/algorithms"
 	"github.com/arduino/arduino-cli/internal/arduino/globals"
+	"github.com/arduino/arduino-cli/internal/arduino/libraries"
 	"github.com/arduino/arduino-cli/internal/i18n"
 	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
 	"github.com/arduino/go-paths-helper"
 )
 
 // Sketch holds all the files composing a sketch
 type Sketch struct {
-	Name             string
-	MainFile         *paths.Path
-	FullPath         *paths.Path    // FullPath is the path to the Sketch folder
-	OtherSketchFiles paths.PathList // Sketch files that end in .ino other than main file
-	AdditionalFiles  paths.PathList
-	RootFolderFiles  paths.PathList // All files that are in the Sketch root
-	Project          *Project
+	Name              string
+	MainFile          *paths.Path
+	FullPath          *paths.Path    // FullPath is the path to the Sketch folder
+	OtherSketchFiles  paths.PathList // Sketch files that end in .ino other than main file
+	AdditionalFiles   paths.PathList
+	RootFolderFiles   paths.PathList       // All files that are in the Sketch root
+	vendoredLibraries []*libraries.Library // All libraries in the 'libraries' directory in the sketch
+	Project           *Project
 }
 
 // New creates an Sketch instance by reading all the files composing a sketch and grouping them
@@ -142,9 +144,41 @@ func New(path *paths.Path) (*Sketch, error) {
 	sort.Sort(&sketch.OtherSketchFiles)
 	sort.Sort(&sketch.RootFolderFiles)
 
+	// Collect vedndored libraries
+	if librariesPath, ok := sketch.GetVendoredLibrariesDir(); ok {
+		libDirs, err := librariesPath.ReadDir()
+		if err != nil {
+			return nil, fmt.Errorf("%s: %w", i18n.Tr("reading sketch libraries"), err)
+		}
+		libDirs.FilterDirs()
+		for _, libDir := range libDirs {
+			lib, err := libraries.Load(libDir, libraries.Sketch)
+			if err != nil {
+				return nil, fmt.Errorf("%s: %w", i18n.Tr("reading sketch libraries"), err)
+			}
+			sketch.vendoredLibraries = append(sketch.vendoredLibraries, lib)
+		}
+	}
+
 	return sketch, nil
 }
 
+// GetVendoredLibrariesDir returns the 'libraries' directory path.
+// The result is in the res,ok format ok is true if the 'libraries' directory
+// is present in the sketch, false otherwise.
+func (s *Sketch) GetVendoredLibrariesDir() (res *paths.Path, ok bool) {
+	libsDir := s.FullPath.Join("libraries")
+	if libsDir.IsDir() {
+		return libsDir, true
+	}
+	return nil, false
+}
+
+// VendoredLibraries returns the libraries bundled in the sketch' 'libraries' directory.
+func (s *Sketch) VendoredLibraries() []*libraries.Library {
+	return s.vendoredLibraries
+}
+
 // supportedFiles reads all files recursively contained in Sketch and
 // filter out unneded or unsupported ones and returns them
 func (s *Sketch) supportedFiles() (paths.PathList, error) {

internal/arduino/sketch/sketch_test.go

@@ -381,3 +381,11 @@ func TestSketchWithMultipleSymlinkLoops(t *testing.T) {
 	require.Error(t, err)
 	require.Nil(t, sketch)
 }
+
+func TestSketchWithVendoredLibraries(t *testing.T) {
+	sketchPath := paths.New("testdata", "SketchWithLibraries")
+	sk, err := New(sketchPath)
+	require.NoError(t, err)
+	require.Len(t, sk.vendoredLibraries, 1)
+	require.Equal(t, "MyLib", sk.vendoredLibraries[0].Name)
+}

internal/arduino/sketch/testdata/SketchWithLibraries/SketchWithLibraries.ino

@@ -0,0 +1,6 @@
+#include <MyLib.h>
+
+void setup() {}
+void loop() {
+    myFunction();
+}

internal/arduino/sketch/testdata/SketchWithLibraries/libraries/MyLib/MyLib.h

@@ -0,0 +1,3 @@
+
+void myFunction() {
+}

internal/arduino/sketch/testdata/SketchWithLibraries/libraries/MyLib/library.properties

@@ -0,0 +1 @@
+name=MyLib

internal/cli/feedback/result/rpc.go

@@ -172,6 +172,7 @@ const (
 	LibraryLocationPlatformBuiltin           LibraryLocation = "platform"
 	LibraryLocationReferencedPlatformBuiltin LibraryLocation = "ref-platform"
 	LibraryLocationUnmanged                  LibraryLocation = "unmanaged"
+	LibraryLocationSketch                    LibraryLocation = "sketch"
 )
 
 func NewLibraryLocation(r rpc.LibraryLocation) LibraryLocation {
@@ -186,6 +187,8 @@ func NewLibraryLocation(r rpc.LibraryLocation) LibraryLocation {
 		return LibraryLocationUser
 	case rpc.LibraryLocation_LIBRARY_LOCATION_UNMANAGED:
 		return LibraryLocationUnmanged
+	case rpc.LibraryLocation_LIBRARY_LOCATION_SKETCH:
+		return LibraryLocationSketch
 	}
 	return LibraryLocationIDEBuiltin
 }

internal/integrationtest/arduino-cli.go

@@ -158,6 +158,12 @@ func (cli *ArduinoCLI) SketchbookDir() *paths.Path {
 	return cli.sketchbookDir
 }
 
+// SetSketchbookDir sets the sketchbook directory
+func (cli *ArduinoCLI) SetSketchbookDir(d *paths.Path) {
+	cli.sketchbookDir = d
+	cli.cliEnvVars["ARDUINO_SKETCHBOOK_DIR"] = d.String()
+}
+
 // WorkingDir returns the working directory
 func (cli *ArduinoCLI) WorkingDir() *paths.Path {
 	return cli.workingDir