Scripting: Added WangSet.effectiveTypeForColor

NEWS.md

@@ -4,6 +4,7 @@
 * Removed Space and Ctrl+Space shortcuts from Layers view to avoid conflict with panning (#3672)
 * Scripting: Added API for editing tile layers using terrain sets (with a-morphous, #3758)
 * Scripting: Support erasing tiles in Tool.preview and TileMap.merge
+* Scripting: Added WangSet.effectiveTypeForColor
 * Fixed object preview position with parallax factor on group layer (#3669)
 * Fixed hover highlight rendering with active parallax factor (#3669)
 * Fixed updating of object selection outlines when changing parallax factor (#3669)

docs/scripting-doc/index.d.ts

@@ -3124,6 +3124,17 @@ declare class WangSet extends TiledObject {
    * @since 1.8
    */
   public setColorName(colorIndex: number, name: string) : void
+
+  /**
+   * Returns the effective WangSet type for the given color.
+   *
+   * Always equals the {@link type} of the WangSet in case of corner or edge
+   * sets. In case of a mixed set, it could also be {@link WangSet.Corner} or
+   * {@link WangSet.Edge}, when the given color is only used in that context.
+   *
+   * @since 1.10.2
+   */
+  public effectiveTypeForColor(int color) : typeof WangSet.Edge | typeof WangSet.Corner | typeof WangSet.Mixed
 }
 
 /**

src/libtiled/wangset.cpp

@@ -910,6 +910,36 @@ quint64 WangSet::completeSetSize() const
     }
 }
 
+WangSet::Type WangSet::effectiveTypeForColor(int color) const
+{
+    if (type() == Mixed) {
+        // Determine a meaningful mode by looking at where the color is used.
+        bool usedAsCorner = false;
+        bool usedAsEdge = false;
+
+        if (color > 0 && color <= colorCount()) {
+            for (const WangId wangId : wangIdByTileId()) {
+                for (int i = 0; i < WangId::NumIndexes; ++i) {
+                    if (wangId.indexColor(i) == color) {
+                        const bool isCorner = WangId::isCorner(i);
+                        usedAsCorner |= isCorner;
+                        usedAsEdge |= !isCorner;
+                    }
+                }
+            }
+        }
+
+        if (usedAsEdge == usedAsCorner)
+            return Mixed;
+        else if (usedAsEdge)
+            return Edge;
+        else
+            return Corner;
+    }
+
+    return type();
+}
+
 /**
  * Returns the Nth WangId starting at 0x11111111
  * and, when C is the number of colors, ending at 0xCCCCCCCC.

src/libtiled/wangset.h

@@ -299,6 +299,8 @@ class TILEDSHARED_EXPORT WangSet : public Object
     bool isComplete() const;
     quint64 completeSetSize() const;
 
+    Type effectiveTypeForColor(int color) const;
+
     WangId templateWangIdAt(unsigned n) const;
 
     WangSet *clone(Tileset *tileset) const;

src/tiled/editablewangset.h

@@ -39,7 +39,7 @@ class EditableWangSet : public EditableObject
     Q_PROPERTY(Type type READ type WRITE setType)
     Q_PROPERTY(Tiled::EditableTile *imageTile READ imageTile WRITE setImageTile)
     Q_PROPERTY(int colorCount READ colorCount WRITE setColorCount)
-    Q_PROPERTY(Tiled::EditableTileset *tileset READ tileset)
+    Q_PROPERTY(Tiled::EditableTileset *tileset READ tileset CONSTANT)
 
 public:
     enum Type {
@@ -66,6 +66,8 @@ class EditableWangSet : public EditableObject
     Q_INVOKABLE QString colorName(int colorIndex) const;
     Q_INVOKABLE void setColorName(int colorIndex, const QString &name);
 
+    Q_INVOKABLE Type effectiveTypeForColor(int color) const;
+
     void setName(const QString &name);
     void setType(Type type);
     void setImageTile(Tiled::EditableTile *imageTile);
@@ -100,6 +102,11 @@ inline int EditableWangSet::colorCount() const
     return wangSet()->colorCount();
 }
 
+inline EditableWangSet::Type EditableWangSet::effectiveTypeForColor(int color) const
+{
+    return static_cast<Type>(wangSet()->effectiveTypeForColor(color));
+}
+
 inline WangSet *EditableWangSet::wangSet() const
 {
     return static_cast<WangSet*>(object());

src/tiled/wangbrush.cpp

@@ -214,40 +214,17 @@ void WangBrush::setColor(int color)
     if (!mWangSet)
         return;
 
-    switch (mWangSet->type()) {
+    switch (mWangSet->effectiveTypeForColor(color)) {
     case WangSet::Corner:
         mBrushMode = PaintCorner;
         break;
     case WangSet::Edge:
         mBrushMode = PaintEdge;
         break;
-    case WangSet::Mixed: {
-        // Determine a meaningful mode by looking at where the color is used.
-        bool usedAsCorner = false;
-        bool usedAsEdge = false;
-
-        if (mWangSet && color > 0 && color <= mWangSet->colorCount()) {
-            for (const WangId wangId : mWangSet->wangIdByTileId()) {
-                for (int i = 0; i < WangId::NumIndexes; ++i) {
-                    if (wangId.indexColor(i) == color) {
-                        const bool isCorner = WangId::isCorner(i);
-                        usedAsCorner |= isCorner;
-                        usedAsEdge |= !isCorner;
-                    }
-                }
-            }
-        }
-
-        if (usedAsEdge == usedAsCorner)
-            mBrushMode = PaintEdgeAndCorner;
-        else if (usedAsEdge)
-            mBrushMode = PaintEdge;
-        else
-            mBrushMode = PaintCorner;
-
+    case WangSet::Mixed:
+        mBrushMode = PaintEdgeAndCorner;
         break;
     }
-    }
 }
 
 void WangBrush::mouseMoved(const QPointF &pos, Qt::KeyboardModifiers modifiers)