2- You may have question marks in your head, especially regarding where the probabilities in the Expectation step come from. Please have a look at the explanations on this maths stack exchange .

3- Look at/run this code that I wrote in Python that simulates the solution to the coin-toss problem in the EM tutorial paper of item 1:

P.S The code may be suboptimal, but it does the job.

import numpy as npimport math#### E-M Coin Toss Example as given in the EM tutorial paper by Do and Batzoglou* #### def get_mn_log_likelihood(obs,probs):    """ Return the (log)likelihood of obs, given the probs"""    # Multinomial Distribution Log PMF    # ln (pdf)      =             multinomial coeff            *   product of probabilities    # ln[f(x|n, p)] = [ln(n!) - (ln(x1!)+ln(x2!)+...+ln(xk!))] + [x1*ln(p1)+x2*ln(p2)+...+xk*ln(pk)]         multinomial_coeff_denom= 0    prod_probs = 0    for x in range(0,len(obs)): # loop through state counts in each observation        multinomial_coeff_denom = multinomial_coeff_denom + math.log(math.factorial(obs[x]))        prod_probs = prod_probs + obs[x]*math.log(probs[x])multinomial_coeff = math.log(math.factorial(sum(obs))) -  multinomial_coeff_denomlikelihood = multinomial_coeff + prod_probsreturn likelihood# 1st:  Coin B, {HTTTHHTHTH}, 5H,5T# 2nd:  Coin A, {HHHHTHHHHH}, 9H,1T# 3rd:  Coin A, {HTHHHHHTHH}, 8H,2T# 4th:  Coin B, {HTHTTTHHTT}, 4H,6T# 5th:  Coin A, {THHHTHHHTH}, 7H,3T# so, from MLE: pA(heads) = 0.80 and pB(heads)=0.45# represent the experimentshead_counts = np.array([5,9,8,4,7])tail_counts = 10-head_countsexperiments = zip(head_counts,tail_counts)# initialise the pA(heads) and pB(heads)pA_heads = np.zeros(100); pA_heads[0] = 0.60pB_heads = np.zeros(100); pB_heads[0] = 0.50# E-M begins!delta = 0.001  j = 0 # iteration counterimprovement = float('inf')while (improvement>delta):    expectation_A = np.zeros((5,2), dtype=float)     expectation_B = np.zeros((5,2), dtype=float)    for i in range(0,len(experiments)):        e = experiments[i] # i'th experiment        ll_A = get_mn_log_likelihood(e,np.array([pA_heads[j],1-pA_heads[j]])) # loglikelihood of e given coin A        ll_B = get_mn_log_likelihood(e,np.array([pB_heads[j],1-pB_heads[j]])) # loglikelihood of e given coin B        weightA = math.exp(ll_A) / ( math.exp(ll_A) + math.exp(ll_B) ) # corresponding weight of A proportional to likelihood of A         weightB = math.exp(ll_B) / ( math.exp(ll_A) + math.exp(ll_B) ) # corresponding weight of B proportional to likelihood of B                                    expectation_A[i] = np.dot(weightA, e)         expectation_B[i] = np.dot(weightB, e)    pA_heads[j+1] = sum(expectation_A)[0] / sum(sum(expectation_A));     pB_heads[j+1] = sum(expectation_B)[0] / sum(sum(expectation_B));     improvement = max( abs(np.array([pA_heads[j+1],pB_heads[j+1]]) - np.array([pA_heads[j],pB_heads[j]]) ))    j = j+1

Expectation-Maximization in CSharp

Jump to: ,

This example requires

using System.Drawing;using Emgu.CV.Structure;using Emgu.CV.ML;using Emgu.CV.ML.Structure; ... int N = 4; //number of clustersint N1 = (int)Math.Sqrt((double)4); Bgr[] colors = new Bgr[] {    new Bgr(0, 0, 255),    new Bgr(0, 255, 0),   new Bgr(0, 255, 255),   new Bgr(255, 255, 0)}; int nSamples = 100; Matrix
      
        samples = new Matrix
       
        (nSamples, 2);Matrix
        
          labels = new Matrix
         
          (nSamples, 1);Image
          
            img = new Image
           
            (500, 500);Matrix
            
              sample = new Matrix
             
              (1, 2); CvInvoke.cvReshape(samples.Ptr, samples.Ptr, 2, 0);for (int i = 0; i < N; i++){ Matrix
              
                rows = samples.GetRows(i * nSamples / N, (i + 1) * nSamples / N, 1); double scale = ((i % N1) + 1.0) / (N1 + 1); MCvScalar mean = new MCvScalar(scale * img.Width, scale * img.Height); MCvScalar sigma = new MCvScalar(30, 30); ulong seed = (ulong)DateTime.Now.Ticks; CvInvoke.cvRandArr(ref seed, rows.Ptr, Emgu.CV.CvEnum.RAND_TYPE.CV_RAND_NORMAL, mean, sigma);}CvInvoke.cvReshape(samples.Ptr, samples.Ptr, 1, 0); using (EM emModel1 = new EM())using (EM emModel2 = new EM()){ EMParams parameters1 = new EMParams(); parameters1.Nclusters = N; parameters1.CovMatType = Emgu.CV.ML.MlEnum.EM_COVARIAN_MATRIX_TYPE.COV_MAT_DIAGONAL; parameters1.StartStep = Emgu.CV.ML.MlEnum.EM_INIT_STEP_TYPE.START_AUTO_STEP; parameters1.TermCrit = new MCvTermCriteria(10, 0.01); emModel1.Train(samples, null, parameters1, labels); EMParams parameters2 = new EMParams(); parameters2.Nclusters = N; parameters2.CovMatType = Emgu.CV.ML.MlEnum.EM_COVARIAN_MATRIX_TYPE.COV_MAT_GENERIC; parameters2.StartStep = Emgu.CV.ML.MlEnum.EM_INIT_STEP_TYPE.START_E_STEP; parameters2.TermCrit = new MCvTermCriteria(100, 1.0e-6); parameters2.Means = emModel1.GetMeans(); parameters2.Covs = emModel1.GetCovariances(); parameters2.Weights = emModel1.GetWeights(); emModel2.Train(samples, null, parameters2, labels); #region Classify every image pixel for (int i = 0; i < img.Height; i++) for (int j = 0; j < img.Width; j++) { sample.Data[0, 0] = i; sample.Data[0, 1] = j; int response = (int) emModel2.Predict(sample, null); Bgr color = colors[response]; img.Draw( new CircleF(new PointF(i, j), 1), new Bgr(color.Blue*0.5, color.Green * 0.5, color.Red * 0.5 ), 0); } #endregion #region draw the clustered samples for (int i = 0; i < nSamples; i++) { img.Draw(new CircleF(new PointF(samples.Data[i, 0], samples.Data[i, 1]), 1), colors[labels.Data[i, 0]], 0); } #endregion Emgu.CV.UI.ImageViewer.Show(img);}