Accessing Windows Live Photo Gallery Face API

Projects

While working on a tool to automatically organize media files (move new images to their correct directory) i though: could i use the Windows Live Photo Gallery (WLPG) face recognition engine to my advantage? The photos i want to organize are people, and i usually try to mark the faces with WLPG before running my program (which uses the faces embeded in the jpeg along other metadata to help select the final directory). Maybe i could use the face engine to recognize some faces inside my program.

As it happens, the API is not well prepared to be used by others, and the recognition engine is not perfect, but i managed to use it to some level. This is how i did it using Embarcadero RAD Studio 2010 (Delphi), and with Windows Live Photo Gallery 2011.

First steps

First, you have to import the Type Library “WLPG FaceRecognitionObjects” (which is in WLXFaceRecognition.dll on the WLPG directory). This will give you access to the exposed API.

To process an image file (for detection or recognition) you have to load it into an IWICBitmap object to access its data. I haven’t used exception check code to make it more clear, but you should capture exceptions and make sure you free any objects you create.

var
  theImageManager: IImageManager;
  theFaceWICBitmap: FaceRecognitionObjects_TLB.IWICBitmap;
  theImageData: IImageData;
  theWICImage: Graphics.TWICImage;
begin
  theWICImage := TWICImage.Create;
  theWICImage.LoadFromFile(filename);
  theFaceWICBitmap := FaceRecognitionObjects_TLB.IWICBitmap(theWICImage.Handle);
  theImageManager := CoImageManager.Create;
  theImageManager.CreateImageDataFromWICBitmap(theFaceWICBitmap, theImageData);

The theImageData variable will be used on the following calls to detect or recognize faces.

Face Detection

Detection is an easy task.

var
  theFaceDetection: IFaceDetection;
  theFaceRegionSet: IFaceRegionSet;
  theFaceRegion: IFaceRegion;
  faceCount: Cardinal;
  theFaceRegionRectangle: FaceRegionRect;
  theFacePose: FacePose;
  f: Integer;
begin
  theFaceDetection := CoFaceDetection.Create;
  if theFaceDetection.RunFaceDetection (theImageData, theFaceRegionSet) = S_OK then
    begin
      if theFaceRegionSet <> nil then theFaceRegionSet.GetCount(faceCount);
      for f := 0 to Integer(faceCount) - 1 do
       if theFaceRegionSet.GetAt(f, theFaceRegion) = S_OK then
         begin
           if theFaceRegion.GetFaceRegionRect(theFaceRegionRectangle) = S_OK then
             begin
               // theFaceRegionRectangle contains the face rectangle coordinates (left, top, width, height) in percentages
             end;
           if theFaceRegion.GetFacePose(theFacePose) = S_OK then
             begin
               // theFacePose contains the detected face pose (see the corresponding enumerations in the type library)
             end;
         end;
    end;
end;

For each face you can use the GetFaceRegionRect and GetFacePose methods to get the rectangle and the detected pose (angle of the face). The face objects have other methods but since we only asked to detect the faces they will not return useful data at this point. Unfortunately just detecting a face is not that useful if you cannot recognize it.

Face Recognition

Face recognition is a very different beast to deal with. Lets see first the code.

type
  TFaceRecommendations = array [0 .. 0] of FaceRecommendation;
var
  theFaceRecognitionPipeline: IFaceRecognitionPipeline;
  theFaceRegionSet: IFaceRegionSet;
  theFaceRegion: IFaceRegion;
  faceCount: Cardinal;
  theFaceRegionRectangle: FaceRegionRect;
  theFacePose: FacePose;
  theFacePersonId: Integer;
  theFacePersonName: WideString;
  pFaceRecommendations: ^TFaceRecommendations;
  faceRecommendationsCount: Cardinal;
  f, r: Integer;
begin
  theFaceRecognitionPipeline := CoFaceRecognitionPipeline.Create;
  theFaceRecognitionPipeline.SetMatchThreshold(MatchThresholdDefault);
  if theFaceRecognitionPipeline.RecognizeFacesFromImage (theImageData, theFaceCache, theFaceRegionSet) = S_OK then
    begin
      if theFaceRegionSet <> nil then theFaceRegionSet.GetCount(faceCount);
      for f := 0 to Integer(faceCount) - 1 do
       if theFaceRegionSet.GetAt(f, theFaceRegion) = S_OK then
         begin

           if theFaceRegion.GetFaceRegionRect(theFaceRegionRectangle) = S_OK then
             begin
               // theFaceRegionRectangle contains the face rectangle coordinates (left, top, width, height) in percentages
             end;

           if theFaceRegion.GetFacePose(theFacePose) = S_OK then
             begin
               // theFacePose contains the detected face pose (see the corresponding enumerations in the type library)
             end;

           theFacePersonId := 0;
           if faceregion.GetPersonId(theFacePersonId) <> S_OK then
             begin
               // theFacePersonId contains the internal id of the detected person
             end;

           theFacePersonName := '';
           if faceregion.GetPersonName(theFacePersonName) <> S_OK then
             begin
               // theFacePersonName contains the name of the detected person
             end;

           // Next we read the person recommendations with their confidence level
           pFaceRecommendations := nil; // the method will allocate memory
           if theFaceRegion.GetRecommendations (PUserType3(pFaceRecommendations), faceRecommendationsCount) = S_OK then
             if faceRecommendationsCount > 0 then
               begin
                 for r := 0 to Integer(faceRecommendationsCount) - 1 do
                   if pFaceRecommendations^[r].nPersonId <> 0 then // a personId of 0 means no one.
                     begin
                       // you get the confidence level from pFaceRecommendations^[r].flConfidence (a percentage).
                     end;
               end;

         end;
    end;
end;

It looks very similar to the detection code, but if you look closely you will notice that it only returns a numeric Id for the recognized person(s), and the main method has an additional parameter (theFaceCache). The GetPersonName method only returns something when the engine is 100% sure of the recognition. So, where do we get the person recommendations names? and what is that face cache?

Face exemplar cache

That face cache is the object that holds the faces information, allows the engine to recognize, and maps a person Id with its name. And to properly use the recognized faces from WLPG we will need to populate it from the application database, since the API does not give us access to that data.

We will have to define a class based on the IExemplarCache interface that will hold the face information from the database.

type
  TPersonFaceData = class
  public
    PersonId: cardinal;
    Representation: TBytes;
    FaceRep:array of FaceRepresentation;
    constructor Create(const id: integer; rep: TBytes);
    procedure AddRepresentation (rep:TBytes);
    destructor Destroy; override;
  end;

  TExemplarCache = class(TInterfacedObject, IExemplarCache)
  private
    cache: TObjectList;
    personnames: TStringList;
    statusCacheStamp:integer;
  public
    function GetPersonCount(out puPersonCount: LongWord): HResult; stdcall;
    function GetPersonData(uPersonIndex: LongWord; out pnPersonId: integer; out ppFaceRepresentations: PUserType4; out puRepresentationCount: LongWord) : HResult; stdcall;
    function GetCacheStamp(out pdwCacheStamp: LongWord): HResult; stdcall;

    constructor Create;
    destructor Destroy; override;
    procedure ImportFromWLPG;
    function GetPersonName(uPersonId: integer): string;
  end;

{ TPersonFaceData }

procedure TPersonFaceData.AddRepresentation(rep: TBytes);
begin
  setlength (FaceRep, length(FaceRep)+1);
  FaceRep[high(FaceRep)].uByteCount := length(rep);
  GetMem (FaceRep[high(FaceRep)].pbBytes, FaceRep[high(FaceRep)].uByteCount);
  move (rep[0], FaceRep[high(FaceRep)].pbBytes^, FaceRep[high(FaceRep)].uByteCount);
end;

constructor TPersonFaceData.Create(const id: integer; rep: TBytes);
begin
  PersonId := id;
  Representation := rep;
  setlength (FaceRep, 1);
  FaceRep[0].uByteCount := length(rep);
  GetMem (FaceRep[0].pbBytes, FaceRep[0].uByteCount);
  move (rep[0], FaceRep[0].pbBytes^, FaceRep[0].uByteCount);
end;

destructor TPersonFaceData.Destroy;
var
  I: Integer;
begin
  for I := 0 to high(FaceRep) do
    FreeMem (FaceRep[I].pbBytes);
  inherited;
end;

{ TExemplarCache }

constructor TExemplarCache.Create;
begin
  cache := TObjectList.Create;
  personnames := TStringList.Create;
end;

destructor TExemplarCache.Destroy;
begin
  personnames.Free;
  cache.Free;
  inherited;
end;

function TExemplarCache.GetCacheStamp(out pdwCacheStamp: LongWord): HResult;
begin
  pdwCacheStamp := statusCacheStamp;
  Result := S_OK;
end;

function TExemplarCache.GetPersonCount(out puPersonCount: cardinal): HResult;
begin
  puPersonCount := cache.Count;
  Result := S_OK;
end;

function TExemplarCache.GetPersonData(uPersonIndex: cardinal; out pnPersonId: integer; out ppFaceRepresentations: PUserType4; out puRepresentationCount: cardinal): HResult;
begin
  if (uPersonIndex < cache.Count) then
    begin
      pnPersonId := cache[uPersonIndex].PersonId;
      puRepresentationCount:=length(cache[uPersonIndex].FaceRep);
      ppFaceRepresentations := @(cache[uPersonIndex].FaceRep[0]);
      Result := S_OK;
    end
  else
    begin
      pnPersonId := 0;
      ppFaceRepresentations:=nil;
      puRepresentationCount:=0;
      Result := S_FALSE;
    end;
end;

function TExemplarCache.GetPersonName(uPersonId: integer): string;
var
  i: Integer;
begin
  if (uPersonId > 0) and (uPersonId-1 < personnames.Count) then
    begin
      i:=PersonNames.IndexOfObject(pointer(uPersonId));
      if i >= 0 then Result := PersonNames[i] else Result:='';
    end
  else
    Result := '';
end;

procedure TExemplarCache.ImportFromWLPG;
var
  conn: TADOConnection;
  query: TADOQuery;
  pName: TField;
  pId: TField;
  pData: TField;
  I: Integer;
  found: Boolean;
  wlpg_dir: String;
begin
  cache.Clear;
  personnames.Clear;

  try
    wlpg_dir:=GetSpecialFolderLocation(CSIDL_LOCAL_APPDATA)+'\Microsoft\Windows Live Photo Gallery\';

    // read face exemplar data
    conn:=TADOConnection.Create(nil);
    query:=TADOQuery.Create(nil);
    try
      conn.ConnectionString := 'Provider=Microsoft.SQLLITE.MOBILE.OLEDB.3.0;Data Source="'+wlpg_dir+'FaceExemplars.ed1";SSCE:Max Database Size=4000';
      conn.LoginPrompt := false;
      query.Connection := conn;
      query.CursorLocation := clUseServer;
      query.CursorType := ctOpenForwardOnly;
      query.LockType := ltReadOnly;
      query.SQL.Text := 'SELECT PersonId, Data FROM tblExemplar';
      query.Open;
      try
        while not query.Eof do
          begin
            try
              pId := query.FieldByName('PersonId');
              pData := query.FieldByName('Data');
              if (pId <> nil) and (pData <> nil) then
                begin
                  found := false;
                  for I := 0 to cache.Count - 1 do
                    if cache[i].PersonId = pId.AsInteger then
                      begin
                        found := true;
                        cache[i].AddRepresentation (pData.AsBytes);
                        break;
                      end;
                  if not found then cache.Add (TPersonFaceData.Create (pId.AsInteger, pData.AsBytes));
                end;
            except
            end;
            query.Next;
            application.processmessages;
          end;
      finally
        query.Close;
      end;
      query.SQL.Text := 'SELECT ExemplarCacheStamp FROM tblStatus';
      query.Open;
      try
        if not query.Eof then
          statusCacheStamp := query.FieldByName('ExemplarCacheStamp').AsInteger;
      finally
        query.Close;
      end;
    finally
      query.Free;
      conn.Close;
      conn.Free;
    end;

    // read person names
    conn:=TADOConnection.Create(nil);
    query:=TADOQuery.Create(nil);
    try
      conn.ConnectionString := 'Provider=Microsoft.SQLLITE.MOBILE.OLEDB.3.0;Data Source="'+wlpg_dir+'Pictures.pd6";SSCE:Max Database Size=4000';
      conn.LoginPrompt := false;
      query.Connection := conn;
      query.CursorType := ctOpenForwardOnly;
      query.LockType := ltReadOnly;
      query.CursorLocation := clUseServer;
      query.SQL.Text := 'SELECT PersonId, Name FROM tblPerson';
      query.Open;
      try
        while not query.Eof do
          begin
            try
              pName := query.FieldByName('Name');
              pId := query.FieldByName('PersonId');
              if (pName <> nil) and (pId <> nil) then
                if pId.AsInteger <> 0 then
                  personnames.AddObject (pName.AsString, pointer(pId.AsInteger));
            except
            end;
            query.Next;
            application.processmessages;
          end;
      finally
        query.Close;
        conn.Close;
      end;
    finally
      query.Free;
      conn.Free;
    end;
  except
    on e:exception do
      begin
        cache.Clear;
        personnames.Clear;
      end;
  end;
end;

An object of type TExemplarCache, populated using the ImportFromWLPG method will be passed to the RecognizeFacesFromImage method, which will use its data to recognize the faces. Then, for each face we can use the GetPersonName method of the cache to know the name associated to the id returned.

Unfortunately, the recognition engine is very slow and needs to be properly trained (within WLPG) to return good confidence levels for the faces.

It is recommended to not run WLPG at the same time you call the ImportFromWLPG method. The code could raise an exception because the database is already in use.

UPDATE: My WLPG database got corrupted. Not sure if it was because i ran some tests while having WLPG open. Thankfully, i had a backup. I suggest you make sure WLPG is not running, or make backups of the entire database directory.

I hope Microsoft will open more their API in a next version, to give easier and greater access to the face exemplar cache.

, , , ,