Cast spells with gestures your UNO learns

Game consoles with motion controllers and smartwatches that recognise a wrist flick all have one thing in common: they recognise gestures. Nobody writes rules like "a flick is when the x value goes above 1.3". Instead, the device is shown examples and learns what each gesture looks like. In this lesson you do exactly that with a motion sensor and the UNO. You record a few examples of three gestures: fire, ice and wind. The UNO learns with the k-nearest neighbours method. Then you play Spell caster in the Arduino Playground: monsters walk towards your tower, each showing the spell that beats it, and you hold the button and make the gesture. The UNO says which spell it recognised and how sure it is.

What you'll learn

• How to turn a movement into a few numbers (features): mean, spread and range
• How k-nearest neighbours classifies a new example by comparing it with stored ones
• Why you need several examples per gesture, and how to tell when the AI is unsure
• How to save the training in EEPROM, so the UNO remembers it

What you need

• UNO R3 board and USB cable (UNO R3 Starter Kit)
• GY-521 MPU-6050 module (45-in-1 Sensor Kit)
• 1 push button (UNO R3 Starter Kit)
• 4 long jumper wires, or a small box to hold the sensor and button like a wand
• A computer with Chrome or Edge
• Breadboard and jumper wires (UNO R3 Starter Kit)

Wiring

Hold the sensor like a wand handle: tape it to a ruler, a pencil or a small box, with the button where your thumb is. The way you hold it must stay the same while training and playing.

The sketch

Wire and EEPROM come with the Arduino IDE.

#include <Wire.h>
#include <EEPROM.h>

const int MPU = 0x68;

void wakeMpu() {
  Wire.begin();
  Wire.beginTransmission(MPU);
  Wire.write(0x6B);                      // power management: wake up
  Wire.write(0);
  Wire.endTransmission();
}

void readAccel(float& ax, float& ay, float& az) {   // in g
  Wire.beginTransmission(MPU);
  Wire.write(0x3B);
  Wire.endTransmission(false);
  Wire.requestFrom(MPU, 6);
  int16_t v[3];
  for (int i = 0; i < 3; i++) {
    v[i] = Wire.read() << 8;
    v[i] |= Wire.read();
  }
  ax = v[0] / 16384.0;
  ay = v[1] / 16384.0;
  az = v[2] / 16384.0;
}

const int BUTTON_PIN = 2, LED_PIN = 13;
const int SPELLS = 3, PER_SPELL = 6, FEATURES = 9;
const char* NAMES[SPELLS] = {"fire", "ice", "wind"};
const float TOO_FAR = 1.2;               // further than this from every example = "I don't know"

struct Memory {
  byte marker;                           // 0x47 = training saved
  byte count[SPELLS];                    // examples per spell
  float ex[SPELLS][PER_SPELL][FEATURES];
};
Memory mem;
int trainSpell = -1;                     // -1 = play mode, 0..2 = record examples of that spell

// Record a gesture while the button is held (at most 1.5 s) and turn it into 9 numbers
bool recordGesture(float* f) {           // f: room for FEATURES numbers
  float sum[3] = {0}, sq[3] = {0}, lo[3] = {9, 9, 9}, hi[3] = {-9, -9, -9};
  int n = 0;
  unsigned long start = millis();
  digitalWrite(LED_PIN, HIGH);
  while (digitalRead(BUTTON_PIN) == LOW && millis() - start < 1500) {
    float a[3];
    readAccel(a[0], a[1], a[2]);
    for (int i = 0; i < 3; i++) {
      sum[i] += a[i];
      sq[i] += a[i] * a[i];
      lo[i] = min(lo[i], a[i]);
      hi[i] = max(hi[i], a[i]);
    }
    n++;
    delay(20);                           // 50 samples a second
  }
  digitalWrite(LED_PIN, LOW);
  if (n < 10) return false;              // too short to be a gesture
  for (int i = 0; i < 3; i++) {
    float mean = sum[i] / n;
    f[i] = mean;                                             // which way the wand points
    f[3 + i] = sqrt(max(sq[i] / n - mean * mean, 0.0f));     // how much it shakes
    f[6 + i] = (hi[i] - lo[i]) / 2;                          // how far it swings
  }
  return true;
}

float distance(const float* a, const float* b) {
  float d = 0;
  for (int i = 0; i < FEATURES; i++) d += (a[i] - b[i]) * (a[i] - b[i]);
  return sqrt(d);
}

// k-nearest neighbours with k = 3: the 3 closest examples vote
void classify(const float* f) {
  float bestD[3] = {1e9, 1e9, 1e9};
  int bestS[3] = {-1, -1, -1};
  for (int s = 0; s < SPELLS; s++) {
    for (int e = 0; e < mem.count[s]; e++) {
      float d = distance(f, mem.ex[s][e]);
      for (int k = 0; k < 3; k++) {      // keep the 3 smallest distances, in order
        if (d < bestD[k]) {
          for (int j = 2; j > k; j--) { bestD[j] = bestD[j - 1]; bestS[j] = bestS[j - 1]; }
          bestD[k] = d;
          bestS[k] = s;
          break;
        }
      }
    }
  }
  if (bestS[0] < 0 || bestD[0] > TOO_FAR) {
    Serial.println("spell:none,c:0");
    return;
  }
  int votes[SPELLS] = {0};
  int used = 0;
  for (int k = 0; k < 3; k++) if (bestS[k] >= 0) { votes[bestS[k]]++; used++; }
  int winner = bestS[0];                 // on a tie the nearest example wins
  for (int s = 0; s < SPELLS; s++) if (votes[s] > votes[winner]) winner = s;
  Serial.print("spell:");
  Serial.print(NAMES[winner]);
  Serial.print(",c:");
  Serial.println(votes[winner] * 100 / used);
}

void printCounts() {
  Serial.print("train:counts");
  for (int s = 0; s < SPELLS; s++) {
    Serial.print(",");
    Serial.print(NAMES[s]);
    Serial.print(":");
    Serial.print(mem.count[s]);
  }
  Serial.println();
}

void setup() {
  Serial.begin(115200);
  pinMode(BUTTON_PIN, INPUT_PULLUP);
  pinMode(LED_PIN, OUTPUT);
  wakeMpu();
  EEPROM.get(0, mem);
  if (mem.marker != 0x47) {              // nothing learned yet
    memset(&mem, 0, sizeof(mem));
    mem.marker = 0x47;
  }
  Serial.println("info:Type 1, 2 or 3 to train fire, ice or wind. Type p to play, c to clear.");
  printCounts();
}

void loop() {
  if (Serial.available()) {              // commands from the Serial Monitor
    char c = Serial.read();
    if (c >= '1' && c <= '3') {
      trainSpell = c - '1';
      Serial.print("info:Training ");
      Serial.print(NAMES[trainSpell]);
      Serial.println(". Hold the button and make the gesture.");
    } else if (c == 'p') {
      trainSpell = -1;
      Serial.println("info:Play mode");
    } else if (c == 'c') {
      memset(mem.count, 0, sizeof(mem.count));
      EEPROM.put(0, mem);
      printCounts();
    }
  }

  if (digitalRead(BUTTON_PIN) == LOW) {
    float f[FEATURES];
    if (recordGesture(f)) {
      if (trainSpell >= 0) {
        int s = trainSpell;
        int slot = mem.count[s] < PER_SPELL ? mem.count[s]++ : random(PER_SPELL);   // full: replace one
        memcpy(mem.ex[s][slot], f, sizeof(f[0]) * FEATURES);
        EEPROM.put(0, mem);
        printCounts();
      } else {
        classify(f);
      }
    }
    while (digitalRead(BUTTON_PIN) == LOW) delay(10);   // wait for release
  }
}

Train your three spells

1. Upload the sketch and open the Serial Monitor at 115200 baud. Set the line ending to "No line ending".
2. Choose three gestures that feel really different. For example: fire = a quick flick upwards, ice = a slow twist of the wrist, wind = a fast shake from side to side.
3. Type 1 and press Enter to train fire. Hold the button down, make the gesture, and let go. Do it 5 or 6 times, a little differently each time. The counts go up.
4. Type 2 for ice and 3 for wind, and record 5 or 6 examples of each.
5. Type p for play mode and test: hold the button, make a gesture, let go. The Serial Monitor shows for example spell:fire,c:100. If it often guesses wrong, type c to clear everything and choose gestures that are more different.

Play Spell caster

1. Close the Serial Monitor: only one program can use the USB port at a time.
2. Open the Arduino Playground in Chrome or Edge on a computer and click Connect your board. Pick the UNO's port and click Connect.
3. Connecting restarts the UNO, but what it learned is saved in its EEPROM, so it remembers everything.

Every monster carries the symbol of the spell that defeats it. Hold the button, make that gesture and release. A recognised spell appears at the top with its confidence; the nearest matching monster disappears.

How it works

A gesture is a stream of hundreds of numbers, which is far too many to compare directly. So the sketch summarises each gesture into 9 features: for each direction (x, y and z), the mean (the average direction the sensor pointed), the standard deviation (how much it shook) and the range (how far it swung). A twist and a flick may take the same time, but their features look very different.

k-nearest neighbours is one of the simplest learning methods there is, and one of the most useful. Training is just storing examples. To classify a new gesture, the UNO measures the distance from its 9 features to every stored example, like measuring the distance between two points, but in 9 dimensions instead of 2. The 3 nearest examples vote, and the spell with the most votes wins. Three out of three votes gives 100 % confidence; two out of three gives 67 %.

If even the nearest example is far away (TOO_FAR), the gesture doesn't look like anything the UNO has seen, so it says "none" instead of guessing. Good AI systems should know when they don't know. Everything is saved in the EEPROM, so the UNO keeps its training when the Playground connects (which restarts it) or when you unplug it.

Try this next

• Change k from 3 to 1 (only the nearest example decides) and test whether the game gets better or worse.
• Let a friend play with your training. Does it still work? Then train with examples from both of you.
• Add a fourth spell. You will need a fourth name, more EEPROM space (check sizeof(Memory)) and a fourth key.
• Print the 9 features of every gesture and look at them: which numbers really separate your spells?

Troubleshooting

• It always says the same spell: your three gestures are too similar. Clear with c and choose very different movements.
• "spell:none" every time: you hold the sensor differently from when you trained it. Keep the same grip, or raise TOO_FAR to 2.
• Typing 1, 2 or 3 does nothing: set the Serial Monitor to 115200 baud and "No line ending".
• "Could not connect": another program is using the port. Close the Serial Monitor and the Serial Plotter, then click Connect again.