"""README,Author-AnuragKumar(mailto:anuragkumarak95@gmail.com)Requirements:-sklearn-numpy-matplotlibPython:-3.5Inputs:-X,a2Dnumpyarrayoffeatures.-k,numberofclusterstocreate.-initial_centroids,initialcentroidvaluesgeneratedbyutilityfunction(mentionedinusage).-maxiter,maximumnumberofiterationstoprocess.-heterogeneity,emptylistthatwillbefilledwithhetrogeneityvaluesifpassedtokmeansfunc.Usage:1.define'k'value,'X'featuresarrayand'hetrogeneity'emptylist2.createinitial_centroids,initial_centroids=get_initial_centroids(X,k,seed=0#seedvalueforinitialcentroidgeneration,#Noneforrandomness(default=None))3.findcentroidsandclustersusingkmeansfunction.centroids,cluster_assignment=kmeans(X,k,initial_centroids,maxiter=400,record_heterogeneity=heterogeneity,verbose=True#whethertoprintlogsinconsoleornot.(default=False))4.Plotthelossfunction,hetrogeneityvaluesforeveryiterationsavedinhetrogeneitylist.plot_heterogeneity(heterogeneity,k)5.TransfersDataframeintoexcelformatitmusthavefeaturecalled'Clust'withkmeansclusteringnumbersinit."""importwarningsimportnumpyasnpimportpandasaspdfrommatplotlibimportpyplotaspltfromsklearn.metricsimportpairwise_distanceswarnings.filterwarnings("ignore")TAG="K-MEANS-CLUST/"defget_initial_centroids(data,k,seed=None):"""Randomlychoosekdatapointsasinitialcentroids"""ifseedisnotNone:#usefulforobtainingconsistentresultsnp.random.seed(seed)n=data.shape[0]#numberofdatapoints#PickKindicesfromrange[0,N).rand_indices=np.random.randint(0,n,k)#Keepcentroidsasdenseformat,asmanyentrieswillbenonzeroduetoaveraging.#Aslongasatleastonedocumentinaclustercontainsaword,#itwillcarryanonzeroweightintheTF-IDFvectorofthecentroid.centroids=data[rand_indices,:]returncentroidsdefcentroid_pairwise_dist(X,centroids):returnpairwise_distances(X,centroids,metric="euclidean")defassign_clusters(data,centroids):#Computedistancesbetweeneachdatapointandthesetofcentroids:#Fillintheblank(RHSonly)distances_from_centroids=centroid_pairwise_dist(data,centroids)#Computeclusterassignmentsforeachdatapoint:#Fillintheblank(RHSonly)cluster_assignment=np.argmin(distances_from_centroids,axis=1)returncluster_assignmentdefrevise_centroids(data,k,cluster_assignment):new_centroids=[]foriinrange(k):#Selectalldatapointsthatbelongtoclusteri.Fillintheblank(RHSonly)member_data_points=data[cluster_assignment==i]#Computethemeanofthedatapoints.Fillintheblank(RHSonly)centroid=member_data_points.mean(axis=0)new_centroids.append(centroid)new_centroids=np.array(new_centroids)returnnew_centroidsdefcompute_heterogeneity(data,k,centroids,cluster_assignment):heterogeneity=0.0foriinrange(k):#Selectalldatapointsthatbelongtoclusteri.Fillintheblank(RHSonly)member_data_points=data[cluster_assignment==i,:]ifmember_data_points.shape[0]>0:#checkifi-thclusterisnon-empty#Computedistancesfromcentroidtodatapoints(RHSonly)distances=pairwise_distances(member_data_points,[centroids[i]],metric="euclidean")squared_distances=distances**2heterogeneity+=np.sum(squared_distances)returnheterogeneitydefplot_heterogeneity(heterogeneity,k):plt.figure(figsize=(7,4))plt.plot(heterogeneity,linewidth=4)plt.xlabel("#Iterations")plt.ylabel("Heterogeneity")plt.title(f"Heterogeneityofclusteringovertime,K={k:d}")plt.rcParams.update({"font.size":16})plt.show()defkmeans(data,k,initial_centroids,maxiter=500,record_heterogeneity=None,verbose=False):"""Thisfunctionrunsk-meansongivendataandinitialsetofcentroids.maxiter:maximumnumberofiterationstorun.(default=500)record_heterogeneity:(optional)alist,tostorethehistoryofheterogeneityasfunctionofiterationsifNone,donotstorethehistory.verbose:ifTrue,printhowmanydatapointschangedtheirclusterlabelsineachiteration"""centroids=initial_centroids[:]prev_cluster_assignment=Noneforitrinrange(maxiter):ifverbose:print(itr,end="")#1.Makeclusterassignmentsusingnearestcentroidscluster_assignment=assign_clusters(data,centroids)#2.Computeanewcentroidforeachofthekclusters,averagingalldata#pointsassignedtothatcluster.centroids=revise_centroids(data,k,cluster_assignment)#Checkforconvergence:ifnoneoftheassignmentschanged,stopif(prev_cluster_assignmentisnotNoneand(prev_cluster_assignment==cluster_assignment).all()):break#Printnumberofnewassignmentsifprev_cluster_assignmentisnotNone:num_changed=np.sum(prev_cluster_assignment!=cluster_assignment)ifverbose:print("{:5d}elementschangedtheirclusterassignment.".format(num_changed))#Recordheterogeneityconvergencemetricifrecord_heterogeneityisnotNone:#YOURCODEHEREscore=compute_heterogeneity(data,k,centroids,cluster_assignment)record_heterogeneity.append(score)prev_cluster_assignment=cluster_assignment[:]returncentroids,cluster_assignment#MocktestbelowifFalse:#changetotruetorunthistestcase.fromsklearnimportdatasetsasdsdataset=ds.load_iris()k=3heterogeneity=[]initial_centroids=get_initial_centroids(dataset["data"],k,seed=0)centroids,cluster_assignment=kmeans(dataset["data"],k,initial_centroids,maxiter=400,record_heterogeneity=heterogeneity,verbose=True,)plot_heterogeneity(heterogeneity,k)defReportGenerator(df:pd.DataFrame,ClusteringVariables:np.array,FillMissingReport=None)->pd.DataFrame:"""Functiongenerateseasy-eradingclusteringreport.Ittakes2argumentsasaninput:DataFrame-dataframewithpredictedcluestercolumn;FillMissingReport-dictionaryofruleshowwearegoingtofillmissingvaluesofforfinalreportgenerate(notincludedinmodeling);inordertorunthefunctionfollowinglibrariesmustbeimported:importpandasaspdimportnumpyasnp>>>data=pd.DataFrame()>>>data['numbers']=[1,2,3]>>>data['col1']=[0.5,2.5,4.5]>>>data['col2']=[100,200,300]>>>data['col3']=[10,20,30]>>>data['Cluster']=[1,1,2]>>>ReportGenerator(data,['col1','col2'],0)FeaturesTypeMark120#ofCustomersClusterSizeFalse2.0000001.0000001%ofCustomersClusterProportionFalse0.6666670.3333332col1mean_with_zerosTrue1.5000004.5000003col2mean_with_zerosTrue150.000000300.0000004numbersmean_with_zerosFalse1.5000003.000000.................99dummy5%False1.0000001.000000100dummy95%False1.0000001.000000101dummystdevFalse0.000000NaN102dummymodeFalse1.0000001.000000103dummymedianFalse1.0000001.000000[104rowsx5columns]"""#FillmissingvalueswithgivenrulesifFillMissingReport:df.fillna(value=FillMissingReport,inplace=True)df["dummy"]=1numeric_cols=df.select_dtypes(np.number).columnsreport=(df.groupby(["Cluster"])[#constructreportdataframenumeric_cols]#groupbyclusternumber.agg([("sum",np.sum),("mean_with_zeros",lambdax:np.mean(np.nan_to_num(x))),("mean_without_zeros",lambdax:x.replace(0,np.NaN).mean()),("mean_25-75",lambdax:np.mean(np.nan_to_num(sorted(x)[round(len(x)*25/100):round(len(x)*75/100)])),),("mean_with_na",np.mean),("min",lambdax:x.min()),("5%",lambdax:x.quantile(0.05)),("25%",lambdax:x.quantile(0.25)),("50%",lambdax:x.quantile(0.50)),("75%",lambdax:x.quantile(0.75)),("95%",lambdax:x.quantile(0.95)),("max",lambdax:x.max()),("count",lambdax:x.count()),("stdev",lambdax:x.std()),("mode",lambdax:x.mode()[0]),("median",lambdax:x.median()),("#>0",lambdax:(x>0).sum()),]).T.reset_index().rename(index=str,columns={"level_0":"Features","level_1":"Type"}))#renamecolumns#calculatethesizeofcluster(countofclientID's)clustersize=report[(report["Features"]=="dummy")&(report["Type"]=="count")].copy()#avoidSettingWithCopyWarningclustersize.Type=("ClusterSize"#renamecreatedclusterdftomatchreportcolumnnames)clustersize.Features="#ofCustomers"clusterproportion=pd.DataFrame(clustersize.iloc[:,2:].values/clustersize.iloc[:,2:].values.sum()#calculatingtheproportionofcluster)clusterproportion["Type"]="%ofCustomers"#renamecreatedclusterdftomatchreportcolumnnamesclusterproportion["Features"]="ClusterProportion"cols=clusterproportion.columns.tolist()cols=cols[-2:]+cols[:-2]clusterproportion=clusterproportion[cols]#rearrangecolumnstomatchreportclusterproportion.columns=report.columnsa=pd.DataFrame(abs(report[report["Type"]=="count"].iloc[:,2:].values-clustersize.iloc[:,2:].values))#generatingdfwithcountofnanvaluesa["Features"]=0a["Type"]="#ofnan"a.Features=report[report["Type"]=="count"].Features.tolist()#fillingvaluesinordertomatchreportcols=a.columns.tolist()cols=cols[-2:]+cols[:-2]a=a[cols]#rearrangecolumnstomatchreporta.columns=report.columns#renamecolumnstomatchreportreport=report.drop(report[report.Type=="count"].index)#dropcountvaluesexceptclustersizereport=pd.concat([report,a,clustersize,clusterproportion],axis=0)#concatreportwithclustertsizeandnanvaluesreport["Mark"]=report["Features"].isin(ClusteringVariables)cols=report.columns.tolist()cols=cols[0:2]+cols[-1:]+cols[2:-1]report=report[cols]sorter1={"ClusterSize":9,"ClusterProportion":8,"mean_with_zeros":7,"mean_with_na":6,"max":5,"50%":4,"min":3,"25%":2,"75%":1,"#ofnan":0,"#>0":-1,"sum_with_na":-2,}report=(report.assign(Sorter1=lambdax:x.Type.map(sorter1),Sorter2=lambdax:list(reversed(range(len(x)))),).sort_values(["Sorter1","Mark","Sorter2"],ascending=False).drop(["Sorter1","Sorter2"],axis=1))report.columns.name=""report=report.reset_index()report.drop(columns=["index"],inplace=True)returnreportif__name__=="__main__":importdoctestdoctest.testmod()
×
Hi there! Thanks for stumbling upon my website!
- Hussain Omer
Scroll Down ↓
"""README,Author-AnuragKumar(mailto:anuragkumarak95@gmail.com)Requirements:-sklearn-numpy-matplotlibPython:-3.5Inputs:-X,a2Dnumpyarrayoffeatures.-k,numberofclusterstocreate.-initial_centroids,initialcentroidvaluesgeneratedbyutilityfunction(mentionedinusage).-maxiter,maximumnumberofiterationstoprocess.-heterogeneity,emptylistthatwillbefilledwithhetrogeneityvaluesifpassedtokmeansfunc.Usage:1.define'k'value,'X'featuresarrayand'hetrogeneity'emptylist2.createinitial_centroids,initial_centroids=get_initial_centroids(X,k,seed=0#seedvalueforinitialcentroidgeneration,#Noneforrandomness(default=None))3.findcentroidsandclustersusingkmeansfunction.centroids,cluster_assignment=kmeans(X,k,initial_centroids,maxiter=400,record_heterogeneity=heterogeneity,verbose=True#whethertoprintlogsinconsoleornot.(default=False))4.Plotthelossfunction,hetrogeneityvaluesforeveryiterationsavedinhetrogeneitylist.plot_heterogeneity(heterogeneity,k)5.TransfersDataframeintoexcelformatitmusthavefeaturecalled'Clust'withkmeansclusteringnumbersinit."""importwarningsimportnumpyasnpimportpandasaspdfrommatplotlibimportpyplotaspltfromsklearn.metricsimportpairwise_distanceswarnings.filterwarnings("ignore")TAG="K-MEANS-CLUST/"defget_initial_centroids(data,k,seed=None):"""Randomlychoosekdatapointsasinitialcentroids"""ifseedisnotNone:#usefulforobtainingconsistentresultsnp.random.seed(seed)n=data.shape[0]#numberofdatapoints#PickKindicesfromrange[0,N).rand_indices=np.random.randint(0,n,k)#Keepcentroidsasdenseformat,asmanyentrieswillbenonzeroduetoaveraging.#Aslongasatleastonedocumentinaclustercontainsaword,#itwillcarryanonzeroweightintheTF-IDFvectorofthecentroid.centroids=data[rand_indices,:]returncentroidsdefcentroid_pairwise_dist(X,centroids):returnpairwise_distances(X,centroids,metric="euclidean")defassign_clusters(data,centroids):#Computedistancesbetweeneachdatapointandthesetofcentroids:#Fillintheblank(RHSonly)distances_from_centroids=centroid_pairwise_dist(data,centroids)#Computeclusterassignmentsforeachdatapoint:#Fillintheblank(RHSonly)cluster_assignment=np.argmin(distances_from_centroids,axis=1)returncluster_assignmentdefrevise_centroids(data,k,cluster_assignment):new_centroids=[]foriinrange(k):#Selectalldatapointsthatbelongtoclusteri.Fillintheblank(RHSonly)member_data_points=data[cluster_assignment==i]#Computethemeanofthedatapoints.Fillintheblank(RHSonly)centroid=member_data_points.mean(axis=0)new_centroids.append(centroid)new_centroids=np.array(new_centroids)returnnew_centroidsdefcompute_heterogeneity(data,k,centroids,cluster_assignment):heterogeneity=0.0foriinrange(k):#Selectalldatapointsthatbelongtoclusteri.Fillintheblank(RHSonly)member_data_points=data[cluster_assignment==i,:]ifmember_data_points.shape[0]>0:#checkifi-thclusterisnon-empty#Computedistancesfromcentroidtodatapoints(RHSonly)distances=pairwise_distances(member_data_points,[centroids[i]],metric="euclidean")squared_distances=distances**2heterogeneity+=np.sum(squared_distances)returnheterogeneitydefplot_heterogeneity(heterogeneity,k):plt.figure(figsize=(7,4))plt.plot(heterogeneity,linewidth=4)plt.xlabel("#Iterations")plt.ylabel("Heterogeneity")plt.title(f"Heterogeneityofclusteringovertime,K={k:d}")plt.rcParams.update({"font.size":16})plt.show()defkmeans(data,k,initial_centroids,maxiter=500,record_heterogeneity=None,verbose=False):"""Thisfunctionrunsk-meansongivendataandinitialsetofcentroids.maxiter:maximumnumberofiterationstorun.(default=500)record_heterogeneity:(optional)alist,tostorethehistoryofheterogeneityasfunctionofiterationsifNone,donotstorethehistory.verbose:ifTrue,printhowmanydatapointschangedtheirclusterlabelsineachiteration"""centroids=initial_centroids[:]prev_cluster_assignment=Noneforitrinrange(maxiter):ifverbose:print(itr,end="")#1.Makeclusterassignmentsusingnearestcentroidscluster_assignment=assign_clusters(data,centroids)#2.Computeanewcentroidforeachofthekclusters,averagingalldata#pointsassignedtothatcluster.centroids=revise_centroids(data,k,cluster_assignment)#Checkforconvergence:ifnoneoftheassignmentschanged,stopif(prev_cluster_assignmentisnotNoneand(prev_cluster_assignment==cluster_assignment).all()):break#Printnumberofnewassignmentsifprev_cluster_assignmentisnotNone:num_changed=np.sum(prev_cluster_assignment!=cluster_assignment)ifverbose:print("{:5d}elementschangedtheirclusterassignment.".format(num_changed))#Recordheterogeneityconvergencemetricifrecord_heterogeneityisnotNone:#YOURCODEHEREscore=compute_heterogeneity(data,k,centroids,cluster_assignment)record_heterogeneity.append(score)prev_cluster_assignment=cluster_assignment[:]returncentroids,cluster_assignment#MocktestbelowifFalse:#changetotruetorunthistestcase.fromsklearnimportdatasetsasdsdataset=ds.load_iris()k=3heterogeneity=[]initial_centroids=get_initial_centroids(dataset["data"],k,seed=0)centroids,cluster_assignment=kmeans(dataset["data"],k,initial_centroids,maxiter=400,record_heterogeneity=heterogeneity,verbose=True,)plot_heterogeneity(heterogeneity,k)defReportGenerator(df:pd.DataFrame,ClusteringVariables:np.array,FillMissingReport=None)->pd.DataFrame:"""Functiongenerateseasy-eradingclusteringreport.Ittakes2argumentsasaninput:DataFrame-dataframewithpredictedcluestercolumn;FillMissingReport-dictionaryofruleshowwearegoingtofillmissingvaluesofforfinalreportgenerate(notincludedinmodeling);inordertorunthefunctionfollowinglibrariesmustbeimported:importpandasaspdimportnumpyasnp>>>data=pd.DataFrame()>>>data['numbers']=[1,2,3]>>>data['col1']=[0.5,2.5,4.5]>>>data['col2']=[100,200,300]>>>data['col3']=[10,20,30]>>>data['Cluster']=[1,1,2]>>>ReportGenerator(data,['col1','col2'],0)FeaturesTypeMark120#ofCustomersClusterSizeFalse2.0000001.0000001%ofCustomersClusterProportionFalse0.6666670.3333332col1mean_with_zerosTrue1.5000004.5000003col2mean_with_zerosTrue150.000000300.0000004numbersmean_with_zerosFalse1.5000003.000000.................99dummy5%False1.0000001.000000100dummy95%False1.0000001.000000101dummystdevFalse0.000000NaN102dummymodeFalse1.0000001.000000103dummymedianFalse1.0000001.000000[104rowsx5columns]"""#FillmissingvalueswithgivenrulesifFillMissingReport:df.fillna(value=FillMissingReport,inplace=True)df["dummy"]=1numeric_cols=df.select_dtypes(np.number).columnsreport=(df.groupby(["Cluster"])[#constructreportdataframenumeric_cols]#groupbyclusternumber.agg([("sum",np.sum),("mean_with_zeros",lambdax:np.mean(np.nan_to_num(x))),("mean_without_zeros",lambdax:x.replace(0,np.NaN).mean()),("mean_25-75",lambdax:np.mean(np.nan_to_num(sorted(x)[round(len(x)*25/100):round(len(x)*75/100)])),),("mean_with_na",np.mean),("min",lambdax:x.min()),("5%",lambdax:x.quantile(0.05)),("25%",lambdax:x.quantile(0.25)),("50%",lambdax:x.quantile(0.50)),("75%",lambdax:x.quantile(0.75)),("95%",lambdax:x.quantile(0.95)),("max",lambdax:x.max()),("count",lambdax:x.count()),("stdev",lambdax:x.std()),("mode",lambdax:x.mode()[0]),("median",lambdax:x.median()),("#>0",lambdax:(x>0).sum()),]).T.reset_index().rename(index=str,columns={"level_0":"Features","level_1":"Type"}))#renamecolumns#calculatethesizeofcluster(countofclientID's)clustersize=report[(report["Features"]=="dummy")&(report["Type"]=="count")].copy()#avoidSettingWithCopyWarningclustersize.Type=("ClusterSize"#renamecreatedclusterdftomatchreportcolumnnames)clustersize.Features="#ofCustomers"clusterproportion=pd.DataFrame(clustersize.iloc[:,2:].values/clustersize.iloc[:,2:].values.sum()#calculatingtheproportionofcluster)clusterproportion["Type"]="%ofCustomers"#renamecreatedclusterdftomatchreportcolumnnamesclusterproportion["Features"]="ClusterProportion"cols=clusterproportion.columns.tolist()cols=cols[-2:]+cols[:-2]clusterproportion=clusterproportion[cols]#rearrangecolumnstomatchreportclusterproportion.columns=report.columnsa=pd.DataFrame(abs(report[report["Type"]=="count"].iloc[:,2:].values-clustersize.iloc[:,2:].values))#generatingdfwithcountofnanvaluesa["Features"]=0a["Type"]="#ofnan"a.Features=report[report["Type"]=="count"].Features.tolist()#fillingvaluesinordertomatchreportcols=a.columns.tolist()cols=cols[-2:]+cols[:-2]a=a[cols]#rearrangecolumnstomatchreporta.columns=report.columns#renamecolumnstomatchreportreport=report.drop(report[report.Type=="count"].index)#dropcountvaluesexceptclustersizereport=pd.concat([report,a,clustersize,clusterproportion],axis=0)#concatreportwithclustertsizeandnanvaluesreport["Mark"]=report["Features"].isin(ClusteringVariables)cols=report.columns.tolist()cols=cols[0:2]+cols[-1:]+cols[2:-1]report=report[cols]sorter1={"ClusterSize":9,"ClusterProportion":8,"mean_with_zeros":7,"mean_with_na":6,"max":5,"50%":4,"min":3,"25%":2,"75%":1,"#ofnan":0,"#>0":-1,"sum_with_na":-2,}report=(report.assign(Sorter1=lambdax:x.Type.map(sorter1),Sorter2=lambdax:list(reversed(range(len(x)))),).sort_values(["Sorter1","Mark","Sorter2"],ascending=False).drop(["Sorter1","Sorter2"],axis=1))report.columns.name=""report=report.reset_index()report.drop(columns=["index"],inplace=True)returnreportif__name__=="__main__":importdoctestdoctest.testmod()