feat(ui): canvas auto mask followups 3

invokeai/frontend/web/public/locales/en.json

@@ -1889,7 +1889,12 @@
             "reset": "Reset",
             "saveAs": "Save As",
             "cancel": "Cancel",
-            "process": "Process"
+            "process": "Process",
+            "help1": "Auto-mask creates a mask for a single target object. Add <Bold>Include</Bold> and <Bold>Exclude</Bold> points to indicate which parts of the layer are part of the target object.",
+            "help2": "Start with one <Bold>Include</Bold> point within the target object. Add more points to refine the mask. Fewer points typically produce better results.",
+            "clickToAdd": "Click on the layer to add a point",
+            "dragToMove": "Drag a point to move it",
+            "clickToRemove": "Click on a point to remove it"
         },
         "settings": {
             "snapToGrid": {
@@ -1930,6 +1935,8 @@
             "newRegionalReferenceImage": "New Regional Reference Image",
             "newControlLayer": "New Control Layer",
             "newRasterLayer": "New Raster Layer",
+            "newInpaintMask": "New Inpaint Mask",
+            "newRegionalGuidance": "New Regional Guidance",
             "cropCanvasToBbox": "Crop Canvas to Bbox"
         },
         "stagingArea": {

invokeai/frontend/web/src/app/store/middleware/listenerMiddleware/listeners/imageDropped.ts

@@ -8,6 +8,7 @@ import {
   controlLayerAdded,
   entityRasterized,
   entitySelected,
+  inpaintMaskAdded,
   rasterLayerAdded,
   referenceImageAdded,
   referenceImageIPAdapterImageChanged,
@@ -17,6 +18,7 @@ import {
 import { selectCanvasSlice } from 'features/controlLayers/store/selectors';
 import type {
   CanvasControlLayerState,
+  CanvasInpaintMaskState,
   CanvasRasterLayerState,
   CanvasReferenceImageState,
   CanvasRegionalGuidanceState,
@@ -110,6 +112,46 @@ export const addImageDroppedListener = (startAppListening: AppStartListening) =>
         return;
       }
 
+      /**
+
+      /**
+       * Image dropped on Inpaint Mask
+       */
+      if (
+        overData.actionType === 'ADD_INPAINT_MASK_FROM_IMAGE' &&
+        activeData.payloadType === 'IMAGE_DTO' &&
+        activeData.payload.imageDTO
+      ) {
+        const imageObject = imageDTOToImageObject(activeData.payload.imageDTO);
+        const { x, y } = selectCanvasSlice(getState()).bbox.rect;
+        const overrides: Partial<CanvasInpaintMaskState> = {
+          objects: [imageObject],
+          position: { x, y },
+        };
+        dispatch(inpaintMaskAdded({ overrides, isSelected: true }));
+        return;
+      }
+
+      /**
+
+      /**
+       * Image dropped on Regional Guidance
+       */
+      if (
+        overData.actionType === 'ADD_REGIONAL_GUIDANCE_FROM_IMAGE' &&
+        activeData.payloadType === 'IMAGE_DTO' &&
+        activeData.payload.imageDTO
+      ) {
+        const imageObject = imageDTOToImageObject(activeData.payload.imageDTO);
+        const { x, y } = selectCanvasSlice(getState()).bbox.rect;
+        const overrides: Partial<CanvasRegionalGuidanceState> = {
+          objects: [imageObject],
+          position: { x, y },
+        };
+        dispatch(rgAdded({ overrides, isSelected: true }));
+        return;
+      }
+
       /**
        * Image dropped on Raster layer
        */

invokeai/frontend/web/src/features/controlLayers/components/CanvasDropArea.tsx

@@ -41,7 +41,7 @@ export const CanvasDropArea = memo(() => {
   return (
     <>
       <Grid
-        gridTemplateRows="1fr 1fr"
+        gridTemplateRows="1fr 1fr 1fr"
         gridTemplateColumns="1fr 1fr"
         position="absolute"
         top={0}
@@ -62,6 +62,7 @@ export const CanvasDropArea = memo(() => {
             data={addControlLayerFromImageDropData}
           />
         </GridItem>
+
         <GridItem position="relative">
           <IAIDroppable
             dropLabel={t('controlLayers.canvasContextMenu.newRegionalReferenceImage')}

invokeai/frontend/web/src/features/controlLayers/components/InpaintMask/InpaintMask.tsx

@@ -14,7 +14,7 @@ type Props = {
 };
 
 export const InpaintMask = memo(({ id }: Props) => {
-  const entityIdentifier = useMemo<CanvasEntityIdentifier>(() => ({ id, type: 'inpaint_mask' }), [id]);
+  const entityIdentifier = useMemo<CanvasEntityIdentifier<'inpaint_mask'>>(() => ({ id, type: 'inpaint_mask' }), [id]);
 
   return (
     <EntityIdentifierContext.Provider value={entityIdentifier}>

invokeai/frontend/web/src/features/controlLayers/components/RegionalGuidance/RegionalGuidance.tsx

@@ -16,7 +16,10 @@ type Props = {
 };
 
 export const RegionalGuidance = memo(({ id }: Props) => {
-  const entityIdentifier = useMemo<CanvasEntityIdentifier>(() => ({ id, type: 'regional_guidance' }), [id]);
+  const entityIdentifier = useMemo<CanvasEntityIdentifier<'regional_guidance'>>(
+    () => ({ id, type: 'regional_guidance' }),
+    [id]
+  );
 
   return (
     <EntityIdentifierContext.Provider value={entityIdentifier}>

invokeai/frontend/web/src/features/controlLayers/components/SegmentAnything/SegmentAnything.tsx

@@ -3,11 +3,16 @@ import {
   ButtonGroup,
   Flex,
   Heading,
+  Icon,
+  ListItem,
   Menu,
   MenuButton,
   MenuItem,
   MenuList,
   Spacer,
+  Text,
+  Tooltip,
+  UnorderedList,
 } from '@invoke-ai/ui-library';
 import { useStore } from '@nanostores/react';
 import { useAppSelector } from 'app/store/storeHooks';
@@ -20,9 +25,10 @@ import type { CanvasEntityAdapterControlLayer } from 'features/controlLayers/kon
 import type { CanvasEntityAdapterRasterLayer } from 'features/controlLayers/konva/CanvasEntity/CanvasEntityAdapterRasterLayer';
 import { selectAutoProcess } from 'features/controlLayers/store/canvasSettingsSlice';
 import { useRegisteredHotkeys } from 'features/system/components/HotkeysModal/useHotkeyData';
+import type { PropsWithChildren } from 'react';
 import { memo, useCallback, useRef } from 'react';
-import { useTranslation } from 'react-i18next';
-import { PiArrowsCounterClockwiseBold, PiFloppyDiskBold, PiStarBold, PiXBold } from 'react-icons/pi';
+import { Trans, useTranslation } from 'react-i18next';
+import { PiArrowsCounterClockwiseBold, PiFloppyDiskBold, PiInfoBold, PiStarBold, PiXBold } from 'react-icons/pi';
 
 const SegmentAnythingContent = memo(
   ({ adapter }: { adapter: CanvasEntityAdapterRasterLayer | CanvasEntityAdapterControlLayer }) => {
@@ -81,10 +87,17 @@ const SegmentAnythingContent = memo(
         transitionProperty="height"
         transitionDuration="normal"
       >
-        <Flex w="full" gap={4}>
-          <Heading size="md" color="base.300" userSelect="none">
-            {t('controlLayers.segment.autoMask')}
-          </Heading>
+        <Flex w="full" gap={4} alignItems="center">
+          <Flex gap={2}>
+            <Heading size="md" color="base.300" userSelect="none">
+              {t('controlLayers.segment.autoMask')}
+            </Heading>
+            <Tooltip label={<SegmentAnythingHelpTooltipContent />}>
+              <Flex alignItems="center">
+                <Icon as={PiInfoBold} color="base.500" />
+              </Flex>
+            </Tooltip>
+          </Flex>
           <Spacer />
           <CanvasAutoProcessSwitch />
           <CanvasOperationIsolatedLayerPreviewSwitch />
@@ -166,3 +179,31 @@ export const SegmentAnything = () => {
 
   return <SegmentAnythingContent adapter={adapter} />;
 };
+
+const Bold = (props: PropsWithChildren) => (
+  <Text as="span" fontWeight="semibold">
+    {props.children}
+  </Text>
+);
+
+const SegmentAnythingHelpTooltipContent = memo(() => {
+  const { t } = useTranslation();
+
+  return (
+    <Flex gap={3} flexDir="column">
+      <Text>
+        <Trans i18nKey="controlLayers.segment.help1" components={{ Bold: <Bold /> }} />
+      </Text>
+      <Text>
+        <Trans i18nKey="controlLayers.segment.help2" components={{ Bold: <Bold /> }} />
+      </Text>
+      <UnorderedList>
+        <ListItem>{t('controlLayers.segment.clickToAdd')}</ListItem>
+        <ListItem>{t('controlLayers.segment.dragToMove')}</ListItem>
+        <ListItem>{t('controlLayers.segment.clickToRemove')}</ListItem>
+      </UnorderedList>
+    </Flex>
+  );
+});
+
+SegmentAnythingHelpTooltipContent.displayName = 'SegmentAnythingHelpTooltipContent';

invokeai/frontend/web/src/features/controlLayers/components/common/CanvasEntityMenuItemsCopyToClipboard.tsx

@@ -2,6 +2,7 @@ import { MenuItem } from '@invoke-ai/ui-library';
 import { useEntityAdapterSafe } from 'features/controlLayers/contexts/EntityAdapterContext';
 import { useEntityIdentifierContext } from 'features/controlLayers/contexts/EntityIdentifierContext';
 import { useCopyLayerToClipboard } from 'features/controlLayers/hooks/useCopyLayerToClipboard';
+import { useEntityIsEmpty } from 'features/controlLayers/hooks/useEntityIsEmpty';
 import { useIsEntityInteractable } from 'features/controlLayers/hooks/useEntityIsInteractable';
 import { memo, useCallback } from 'react';
 import { useTranslation } from 'react-i18next';
@@ -12,14 +13,15 @@ export const CanvasEntityMenuItemsCopyToClipboard = memo(() => {
   const entityIdentifier = useEntityIdentifierContext();
   const adapter = useEntityAdapterSafe(entityIdentifier);
   const isInteractable = useIsEntityInteractable(entityIdentifier);
+  const isEmpty = useEntityIsEmpty(entityIdentifier);
   const copyLayerToClipboard = useCopyLayerToClipboard();
 
   const onClick = useCallback(() => {
     copyLayerToClipboard(adapter);
   }, [copyLayerToClipboard, adapter]);
 
   return (
-    <MenuItem onClick={onClick} icon={<PiCopyBold />} isDisabled={!isInteractable}>
+    <MenuItem onClick={onClick} icon={<PiCopyBold />} isDisabled={!isInteractable || isEmpty}>
       {t('common.clipboard')}
     </MenuItem>
   );

invokeai/frontend/web/src/features/controlLayers/hooks/addLayerHooks.ts

@@ -24,7 +24,9 @@ import {
   selectEntityOrThrow,
 } from 'features/controlLayers/store/selectors';
 import type {
+  CanvasControlLayerState,
   CanvasEntityIdentifier,
+  CanvasInpaintMaskState,
   CanvasRasterLayerState,
   CanvasRegionalGuidanceState,
   ControlNetConfig,
@@ -124,6 +126,60 @@ export const useNewRasterLayerFromImage = () => {
   return func;
 };
 
+export const useNewControlLayerFromImage = () => {
+  const dispatch = useAppDispatch();
+  const bboxRect = useAppSelector(selectBboxRect);
+  const func = useCallback(
+    (imageDTO: ImageDTO) => {
+      const imageObject = imageDTOToImageObject(imageDTO);
+      const overrides: Partial<CanvasControlLayerState> = {
+        position: { x: bboxRect.x, y: bboxRect.y },
+        objects: [imageObject],
+      };
+      dispatch(controlLayerAdded({ overrides, isSelected: true }));
+    },
+    [bboxRect.x, bboxRect.y, dispatch]
+  );
+
+  return func;
+};
+
+export const useNewInpaintMaskFromImage = () => {
+  const dispatch = useAppDispatch();
+  const bboxRect = useAppSelector(selectBboxRect);
+  const func = useCallback(
+    (imageDTO: ImageDTO) => {
+      const imageObject = imageDTOToImageObject(imageDTO);
+      const overrides: Partial<CanvasInpaintMaskState> = {
+        position: { x: bboxRect.x, y: bboxRect.y },
+        objects: [imageObject],
+      };
+      dispatch(inpaintMaskAdded({ overrides, isSelected: true }));
+    },
+    [bboxRect.x, bboxRect.y, dispatch]
+  );
+
+  return func;
+};
+
+export const useNewRegionalGuidanceFromImage = () => {
+  const dispatch = useAppDispatch();
+  const bboxRect = useAppSelector(selectBboxRect);
+  const func = useCallback(
+    (imageDTO: ImageDTO) => {
+      const imageObject = imageDTOToImageObject(imageDTO);
+      const overrides: Partial<CanvasRegionalGuidanceState> = {
+        position: { x: bboxRect.x, y: bboxRect.y },
+        objects: [imageObject],
+      };
+      dispatch(rgAdded({ overrides, isSelected: true }));
+    },
+    [bboxRect.x, bboxRect.y, dispatch]
+  );
+
+  return func;
+};
+
 /**
  * Returns a function that adds a new canvas with the given image as the initial image, replicating the img2img flow:
  * - Reset the canvas

invokeai/frontend/web/src/features/controlLayers/hooks/useCopyLayerToClipboard.ts

@@ -1,3 +1,4 @@
+import { logger } from 'app/logging/logger';
 import type { CanvasEntityAdapterControlLayer } from 'features/controlLayers/konva/CanvasEntity/CanvasEntityAdapterControlLayer';
 import type { CanvasEntityAdapterInpaintMask } from 'features/controlLayers/konva/CanvasEntity/CanvasEntityAdapterInpaintMask';
 import type { CanvasEntityAdapterRasterLayer } from 'features/controlLayers/konva/CanvasEntity/CanvasEntityAdapterRasterLayer';
@@ -7,6 +8,9 @@ import { copyBlobToClipboard } from 'features/system/util/copyBlobToClipboard';
 import { toast } from 'features/toast/toast';
 import { useCallback } from 'react';
 import { useTranslation } from 'react-i18next';
+import { serializeError } from 'serialize-error';
+
+const log = logger('canvas');
 
 export const useCopyLayerToClipboard = () => {
   const { t } = useTranslation();
@@ -26,11 +30,13 @@ export const useCopyLayerToClipboard = () => {
         const canvas = adapter.getCanvas();
         const blob = await canvasToBlob(canvas);
         copyBlobToClipboard(blob);
+        log.trace('Layer copied to clipboard');
         toast({
           status: 'info',
           title: t('toast.layerCopiedToClipboard'),
         });
       } catch (error) {
+        log.error({ error: serializeError(error) }, 'Problem copying layer to clipboard');
         toast({
           status: 'error',
           title: t('toast.problemCopyingLayer'),

invokeai/frontend/web/src/features/controlLayers/hooks/useEntityIsEmpty.ts

@@ -0,0 +1,11 @@
+import { useAppSelector } from 'app/store/storeHooks';
+import { buildSelectHasObjects } from 'features/controlLayers/store/selectors';
+import type { CanvasEntityIdentifier } from 'features/controlLayers/store/types';
+import { useMemo } from 'react';
+
+export const useEntityIsEmpty = (entityIdentifier: CanvasEntityIdentifier) => {
+  const selectHasObjects = useMemo(() => buildSelectHasObjects(entityIdentifier), [entityIdentifier]);
+  const hasObjects = useAppSelector(selectHasObjects);
+
+  return !hasObjects;
+};

invokeai/frontend/web/src/features/controlLayers/konva/CanvasTool/CanvasEraserToolModule.ts

@@ -216,12 +216,14 @@ export class CanvasEraserToolModule extends CanvasModuleBase {
    */
   onStagePointerDown = async (e: KonvaEventObject<PointerEvent>) => {
     const cursorPos = this.parent.$cursorPos.get();
+    const isPrimaryPointerDown = this.parent.$isPrimaryPointerDown.get();
     const selectedEntity = this.manager.stateApi.getSelectedEntityAdapter();
 
-    if (!cursorPos || !selectedEntity) {
+    if (!cursorPos || !selectedEntity || !isPrimaryPointerDown) {
       /**
        * Can't do anything without:
        * - A cursor position: the cursor is not on the stage
+       * - The mouse is down: the user is not drawing
        * - A selected entity: there is no entity to draw on
        */
       return;

invokeai/frontend/web/src/features/controlLayers/konva/CanvasTool/CanvasToolModule.ts

@@ -160,11 +160,16 @@ export class CanvasToolModule extends CanvasModuleBase {
     const stage = this.manager.stage;
     const tool = this.$tool.get();
     const segmentingAdapter = this.manager.stateApi.$segmentingAdapter.get();
+    const transformingAdapter = this.manager.stateApi.$transformingAdapter.get();
 
-    if ((this.manager.stage.getIsDragging() || tool === 'view') && !segmentingAdapter) {
+    if (this.manager.stage.getIsDragging()) {
+      this.tools.view.syncCursorStyle();
+    } else if (tool === 'view') {
       this.tools.view.syncCursorStyle();
     } else if (segmentingAdapter) {
       segmentingAdapter.segmentAnything.syncCursorStyle();
+    } else if (transformingAdapter) {
+      // The transformer handles cursor style via events
     } else if (this.manager.stateApi.$isFiltering.get()) {
       stage.setCursor('not-allowed');
     } else if (this.manager.stagingArea.$isStaging.get()) {