Environnement - Packages¶
In [27]:
# activer l'environnement
using Pkg
Pkg.activate("env_mlj_comparison")
Activating project at `c:\Users\ricco\Desktop\demo\env_mlj_comparison`
In [28]:
# liste des packages installés
Pkg.status()
Status `C:\Users\ricco\Desktop\demo\env_mlj_comparison\Project.toml` [324d7699] CategoricalArrays v1.1.0 [a93c6f00] DataFrames v1.8.2 [7806a523] DecisionTree v0.12.4 [f6006082] EvoTrees v0.18.5 [7073ff75] IJulia v1.34.4 [b1bec4e5] LIBSVM v0.8.1 [add582a8] MLJ v0.23.2 [c6f25543] MLJDecisionTreeInterface v0.4.4 [61c7150f] MLJLIBSVMInterface v0.2.2 [6ee0df7b] MLJLinearModels v0.10.1 [91a5bcdd] Plots v1.41.6 [30f210dd] ScientificTypesBase v3.1.0 [2913bbd2] StatsBase v0.34.10 [fdbf4ff8] XLSX v0.11.3
Importation et préparation des données¶
Importation, inspection¶
In [29]:
# packages
import DataFrames as DFR
import XLSX
# lecture des données
df = DFR.DataFrame(XLSX.readtable("./spambase.xlsx"))
# premières lignes
println(DFR.describe(df))
56×7 DataFrame Row │ variable mean min median max nmissing eltype │ Symbol Union… Any Union… Any Int64 DataType ─────┼───────────────────────────────────────────────────────────────────────────────── 1 │ wf_make 0.104553 0.0 0.0 4.54 0 Float64 2 │ wf_address 0.213015 0.0 0.0 14.28 0 Float64 3 │ wf_all 0.280656 0.0 0.0 5.1 0 Float64 4 │ wf_3d 0.0654249 0.0 0.0 42.81 0 Float64 5 │ wf_our 0.312223 0.0 0.0 10.0 0 Float64 6 │ wf_over 0.0959009 0.0 0.0 5.88 0 Float64 7 │ wf_remove 0.114208 0.0 0.0 7.27 0 Float64 8 │ wf_internet 0.105295 0.0 0.0 11.11 0 Float64 9 │ wf_order 0.0900674 0.0 0.0 5.26 0 Float64 10 │ wf_mail 0.239413 0.0 0.0 18.18 0 Float64 11 │ wf_receive 0.059824 0.0 0.0 2.61 0 Float64 12 │ wf_will 0.541702 0.0 0.1 9.67 0 Float64 13 │ wf_people 0.0939296 0.0 0.0 5.55 0 Float64 14 │ wf_report 0.0586264 0.0 0.0 10.0 0 Float64 15 │ wf_addresses 0.0492045 0.0 0.0 4.41 0 Float64 16 │ wf_free 0.248848 0.0 0.0 20.0 0 Float64 17 │ wf_business 0.142586 0.0 0.0 7.14 0 Float64 18 │ wf_email 0.184745 0.0 0.0 9.09 0 Float64 19 │ wf_you 1.6621 0.0 1.31 18.75 0 Float64 20 │ wf_credit 0.085577 0.0 0.0 18.18 0 Float64 21 │ wf_your 0.809761 0.0 0.22 11.11 0 Float64 22 │ wf_font 0.121202 0.0 0.0 17.1 0 Float64 23 │ wf_000 0.101645 0.0 0.0 5.45 0 Float64 24 │ wf_money 0.0942686 0.0 0.0 12.5 0 Float64 25 │ wf_hp 0.549504 0.0 0.0 20.83 0 Float64 26 │ wf_hpl 0.265384 0.0 0.0 16.66 0 Float64 27 │ wf_lab 0.0989155 0.0 0.0 14.28 0 Float64 28 │ wf_labs 0.102852 0.0 0.0 5.88 0 Float64 29 │ wf_telnet 0.0647533 0.0 0.0 12.5 0 Float64 30 │ wf_857 0.0470485 0.0 0.0 4.76 0 Float64 31 │ wf_data 0.0972289 0.0 0.0 18.18 0 Float64 32 │ wf_415 0.0478353 0.0 0.0 4.76 0 Float64 33 │ wf_85 0.105412 0.0 0.0 20.0 0 Float64 34 │ wf_technology 0.0974766 0.0 0.0 7.69 0 Float64 35 │ wf_1999 0.136953 0.0 0.0 6.89 0 Float64 36 │ wf_parts 0.0132015 0.0 0.0 8.33 0 Float64 37 │ wf_pm 0.0786286 0.0 0.0 11.11 0 Float64 38 │ wf_direct 0.0648337 0.0 0.0 4.76 0 Float64 39 │ wf_cs 0.0436666 0.0 0.0 7.14 0 Float64 40 │ wf_meeting 0.132339 0.0 0.0 14.28 0 Float64 41 │ wf_original 0.0460987 0.0 0.0 3.57 0 Float64 42 │ wf_project 0.0791958 0.0 0.0 20.0 0 Float64 43 │ wf_re 0.301224 0.0 0.0 21.42 0 Float64 44 │ wf_edu 0.179824 0.0 0.0 22.05 0 Float64 45 │ wf_table 0.00544447 0.0 0.0 2.17 0 Float64 46 │ wf_conference 0.0318692 0.0 0.0 10.0 0 Float64 47 │ cf_; 0.0385747 0.0 0.0 4.385 0 Float64 48 │ cf_( 0.13903 0.0 0.065 9.752 0 Float64 49 │ cf_[ 0.0169759 0.0 0.0 4.081 0 Float64 50 │ cf_! 0.269071 0.0 0.0 32.478 0 Float64 51 │ cf_$ 0.0758107 0.0 0.0 6.003 0 Float64 52 │ cf_# 0.0442382 0.0 0.0 19.829 0 Float64 53 │ capital_run_length_average 5.19152 1.0 2.276 1102.5 0 Float64 54 │ capital_run_length_longest 52.1728 1 15.0 9989 0 Int64 55 │ capital_run_length_total 283.289 1 95.0 15841 0 Int64 56 │ spam no yes 0 String
In [30]:
# dimension
println(DFR.size(df))
(4601, 56)
In [31]:
# retirer la limitation d'affichage
ENV["LINES"] = 1000
# vérifier le schéma de la base
# et les types scientifiques
import MLJ
MLJ.schema(df)
┌────────────────────────────┬────────────┬─────────┐ │ names │ scitypes │ types │ ├────────────────────────────┼────────────┼─────────┤ │ wf_make │ Continuous │ Float64 │ │ wf_address │ Continuous │ Float64 │ │ wf_all │ Continuous │ Float64 │ │ wf_3d │ Continuous │ Float64 │ │ wf_our │ Continuous │ Float64 │ │ wf_over │ Continuous │ Float64 │ │ wf_remove │ Continuous │ Float64 │ │ wf_internet │ Continuous │ Float64 │ │ wf_order │ Continuous │ Float64 │ │ wf_mail │ Continuous │ Float64 │ │ wf_receive │ Continuous │ Float64 │ │ wf_will │ Continuous │ Float64 │ │ wf_people │ Continuous │ Float64 │ │ wf_report │ Continuous │ Float64 │ │ wf_addresses │ Continuous │ Float64 │ │ wf_free │ Continuous │ Float64 │ │ wf_business │ Continuous │ Float64 │ │ wf_email │ Continuous │ Float64 │ │ wf_you │ Continuous │ Float64 │ │ wf_credit │ Continuous │ Float64 │ │ wf_your │ Continuous │ Float64 │ │ wf_font │ Continuous │ Float64 │ │ wf_000 │ Continuous │ Float64 │ │ wf_money │ Continuous │ Float64 │ │ wf_hp │ Continuous │ Float64 │ │ wf_hpl │ Continuous │ Float64 │ │ wf_lab │ Continuous │ Float64 │ │ wf_labs │ Continuous │ Float64 │ │ wf_telnet │ Continuous │ Float64 │ │ wf_857 │ Continuous │ Float64 │ │ wf_data │ Continuous │ Float64 │ │ wf_415 │ Continuous │ Float64 │ │ wf_85 │ Continuous │ Float64 │ │ wf_technology │ Continuous │ Float64 │ │ wf_1999 │ Continuous │ Float64 │ │ wf_parts │ Continuous │ Float64 │ │ wf_pm │ Continuous │ Float64 │ │ wf_direct │ Continuous │ Float64 │ │ wf_cs │ Continuous │ Float64 │ │ wf_meeting │ Continuous │ Float64 │ │ wf_original │ Continuous │ Float64 │ │ wf_project │ Continuous │ Float64 │ │ wf_re │ Continuous │ Float64 │ │ wf_edu │ Continuous │ Float64 │ │ wf_table │ Continuous │ Float64 │ │ wf_conference │ Continuous │ Float64 │ │ cf_; │ Continuous │ Float64 │ │ cf_( │ Continuous │ Float64 │ │ cf_[ │ Continuous │ Float64 │ │ cf_! │ Continuous │ Float64 │ │ cf_$ │ Continuous │ Float64 │ │ cf_# │ Continuous │ Float64 │ │ capital_run_length_average │ Continuous │ Float64 │ │ capital_run_length_longest │ Count │ Int64 │ │ capital_run_length_total │ Count │ Int64 │ │ spam │ Textual │ String │ └────────────────────────────┴────────────┴─────────┘
Préparation des structures¶
In [32]:
# isoler y et X dans des structures distinctes
y, X = MLJ.unpack(df,==(:spam))
# dimensions
println("Dim. de y = $(DFR.size(y))")
println("Dim. de X = $(DFR.size(X))")
Dim. de y = (4601,) Dim. de X = (4601, 55)
Ajustement du type des variables¶
In [33]:
# convertir y en variable catégorielle pour la rég. logistique
# équivalent du type factor sous R
# utilisation du package CategoricalArrays
import CategoricalArrays as CA
y = CA.categorical(y)
# vérification des modalités
CA.levels(y)
2-element CategoricalArrays.CategoricalArray{String,1,UInt32}:
"no"
"yes"
In [34]:
# fréquences des classes
DFR.combine(DFR.groupby(DFR.DataFrame(y=y),:y),DFR.nrow => :freq)
2×2 DataFrame
| Row | y | freq |
|---|---|---|
| Cat… | Int64 | |
| 1 | no | 2788 |
| 2 | yes | 1813 |
In [35]:
# transformer les X count en variables continues
# utilisation du package ScientificTypesBase
import ScientificTypesBase as STB
X = MLJ.coerce(X,STB.Count => STB.Continuous)
# schéma
MLJ.schema(X)
┌────────────────────────────┬────────────┬─────────┐ │ names │ scitypes │ types │ ├────────────────────────────┼────────────┼─────────┤ │ wf_make │ Continuous │ Float64 │ │ wf_address │ Continuous │ Float64 │ │ wf_all │ Continuous │ Float64 │ │ wf_3d │ Continuous │ Float64 │ │ wf_our │ Continuous │ Float64 │ │ wf_over │ Continuous │ Float64 │ │ wf_remove │ Continuous │ Float64 │ │ wf_internet │ Continuous │ Float64 │ │ wf_order │ Continuous │ Float64 │ │ wf_mail │ Continuous │ Float64 │ │ wf_receive │ Continuous │ Float64 │ │ wf_will │ Continuous │ Float64 │ │ wf_people │ Continuous │ Float64 │ │ wf_report │ Continuous │ Float64 │ │ wf_addresses │ Continuous │ Float64 │ │ wf_free │ Continuous │ Float64 │ │ wf_business │ Continuous │ Float64 │ │ wf_email │ Continuous │ Float64 │ │ wf_you │ Continuous │ Float64 │ │ wf_credit │ Continuous │ Float64 │ │ wf_your │ Continuous │ Float64 │ │ wf_font │ Continuous │ Float64 │ │ wf_000 │ Continuous │ Float64 │ │ wf_money │ Continuous │ Float64 │ │ wf_hp │ Continuous │ Float64 │ │ wf_hpl │ Continuous │ Float64 │ │ wf_lab │ Continuous │ Float64 │ │ wf_labs │ Continuous │ Float64 │ │ wf_telnet │ Continuous │ Float64 │ │ wf_857 │ Continuous │ Float64 │ │ wf_data │ Continuous │ Float64 │ │ wf_415 │ Continuous │ Float64 │ │ wf_85 │ Continuous │ Float64 │ │ wf_technology │ Continuous │ Float64 │ │ wf_1999 │ Continuous │ Float64 │ │ wf_parts │ Continuous │ Float64 │ │ wf_pm │ Continuous │ Float64 │ │ wf_direct │ Continuous │ Float64 │ │ wf_cs │ Continuous │ Float64 │ │ wf_meeting │ Continuous │ Float64 │ │ wf_original │ Continuous │ Float64 │ │ wf_project │ Continuous │ Float64 │ │ wf_re │ Continuous │ Float64 │ │ wf_edu │ Continuous │ Float64 │ │ wf_table │ Continuous │ Float64 │ │ wf_conference │ Continuous │ Float64 │ │ cf_; │ Continuous │ Float64 │ │ cf_( │ Continuous │ Float64 │ │ cf_[ │ Continuous │ Float64 │ │ cf_! │ Continuous │ Float64 │ │ cf_$ │ Continuous │ Float64 │ │ cf_# │ Continuous │ Float64 │ │ capital_run_length_average │ Continuous │ Float64 │ │ capital_run_length_longest │ Continuous │ Float64 │ │ capital_run_length_total │ Continuous │ Float64 │ └────────────────────────────┴────────────┴─────────┘
Partition en TRAIN/TEST¶
In [36]:
# effectif
n = DFR.nrow(X)
println(n)
4601
In [37]:
# identifiants -> échantillonner 601 parmi 4601, sans remise
import StatsBase
idTrain = StatsBase.sample(1:n,601,replace=false)
println(length(idTrain))
601
In [38]:
# et les individus en test
idTest = setdiff(1:n,idTrain)
println(length(idTest))
4000
In [39]:
# structures y et X pour train/test
# par indexation avec les indices
yTrain, yTest = y[idTrain], y[idTest]
XTrain, XTest = X[idTrain,:], X[idTest,:]
# afficher les dimensions pour vérifications
println("Dim. de y = $(DFR.size(yTrain)) et $(DFR.size(yTest))")
println("Dim. de X = $(DFR.size(XTrain)) et $(DFR.size(XTest))")
Dim. de y = (601,) et (4000,) Dim. de X = (601, 55) et (4000, 55)
Comparaison des algorithmes (schéma holdout)¶
Chargement des modèles (à partir des packages spécialisés)¶
In [40]:
# chargement des modèles à comparer
DecisionTreeClassifier = @MLJ.load DecisionTreeClassifier pkg=DecisionTree
RandomForestClassifier = @MLJ.load RandomForestClassifier pkg=DecisionTree
LogisticClassifier = @MLJ.load LogisticClassifier pkg=MLJLinearModels
LinearSVC = @MLJ.load LinearSVC pkg=LIBSVM
EvoTreeClassifier = @MLJ.load EvoTreeClassifier pkg=EvoTrees
┌ Info: For silent loading, specify `verbosity=0`. └ @ Main C:\Users\ricco\.julia\packages\MLJModels\9LbNu\src\loading.jl:159
import MLJDecisionTreeInterface ✔ import MLJDecisionTreeInterface ✔ import MLJLinearModels ✔ import MLJLIBSVMInterface ✔ import EvoTrees ✔
┌ Info: For silent loading, specify `verbosity=0`. └ @ Main C:\Users\ricco\.julia\packages\MLJModels\9LbNu\src\loading.jl:159 ┌ Info: For silent loading, specify `verbosity=0`. └ @ Main C:\Users\ricco\.julia\packages\MLJModels\9LbNu\src\loading.jl:159 ┌ Info: For silent loading, specify `verbosity=0`. └ @ Main C:\Users\ricco\.julia\packages\MLJModels\9LbNu\src\loading.jl:159 ┌ Info: For silent loading, specify `verbosity=0`. └ @ Main C:\Users\ricco\.julia\packages\MLJModels\9LbNu\src\loading.jl:159
EvoTrees.EvoTreeClassifier
Instanciation (paramètres par défaut)¶
In [41]:
# instanciation des modèles et intégration
# dans une structure de type liste de tuples
models = [
("Decision Tree", DecisionTreeClassifier()),
("Random Forest", RandomForestClassifier()),
("Logistic Regression", LogisticClassifier()),
("Gradient Boosting", EvoTreeClassifier())
]
4-element Vector{Tuple{String, MLJModelInterface.Probabilistic}}:
("Decision Tree", DecisionTreeClassifier(max_depth = -1, …))
("Random Forest", RandomForestClassifier(max_depth = -1, …))
("Logistic Regression", LogisticClassifier(lambda = 2.220446049250313e-16, …))
("Gradient Boosting", EvoTreeClassifier(loss = mlogloss, …))
Inspection des paramètres¶
In [42]:
# paramètres de l'arbre par exemple
# instancié avec les paramètres par défaut
println("*** Parametre (par défaut) de << Arbre de décision ***")
for (param,value) in pairs(MLJ.params(DecisionTreeClassifier()))
println("$param = $value")
end
*** Parametre (par défaut) de << Arbre de décision *** max_depth = -1 min_samples_leaf = 1 min_samples_split = 2 min_purity_increase = 0.0 n_subfeatures = 0 post_prune = false merge_purity_threshold = 1.0 display_depth = 5 feature_importance = impurity rng = Random.TaskLocalRNG()
In [43]:
# paramètres pour l'ensemble des algos instaciés
# disponibles dans la liste << models >>
for i in 1:length(models)
println()
println("*** Parametre de << $(models[i][1]) >> ***")
for (param,value) in pairs(MLJ.params(models[i][2]))
println("$param = $value")
end
end
*** Parametre de << Decision Tree >> *** max_depth = -1 min_samples_leaf = 1 min_samples_split = 2 min_purity_increase = 0.0 n_subfeatures = 0 post_prune = false merge_purity_threshold = 1.0 display_depth = 5 feature_importance = impurity rng = Random.TaskLocalRNG() *** Parametre de << Random Forest >> *** max_depth = -1 min_samples_leaf = 1 min_samples_split = 2 min_purity_increase = 0.0 n_subfeatures = -1 n_trees = 100 sampling_fraction = 0.7 feature_importance = impurity rng = Random.TaskLocalRNG() *** Parametre de << Logistic Regression >> *** lambda = 2.220446049250313e-16 gamma = 0.0 penalty = l2 fit_intercept = true penalize_intercept = false scale_penalty_with_samples = true solver = nothing *** Parametre de << Gradient Boosting >> *** loss = mlogloss metric = mlogloss nrounds = 100 bagging_size = 1 early_stopping_rounds = 9223372036854775807 L2 = 1.0 lambda = 0.0 gamma = 0.0 eta = 0.1 max_depth = 6 min_weight = 1.0 rowsample = 1.0 colsample = 1.0 nbins = 64 tree_type = binary seed = 123 device = cpu
Expérimentation¶
In [44]:
# lancer l'expérimentation
# structure de récupération des résultats -> liste
results = []
# pour chaque algo
for (name, model) in models
# préparation - entraînement
mach = MLJ.machine(model, XTrain, yTrain)
MLJ.fit!(mach, verbosity=0)
# prédiction en test
ypred = MLJ.predict(mach, XTest) # proba d'appartenance
ypred = MLJ.mode.(ypred) # convertir probabilités -> classes
# accuracy
acc = MLJ.accuracy(ypred, yTest)
# rajouter le résultat sous la forme de named-tuple
# dans la liste
push!(results, (model=name, accuracy=acc))
end
# transformation de la liste en dataframe
df_results = DFR.DataFrame(results)
4×2 DataFrame
| Row | model | accuracy |
|---|---|---|
| String | Float64 | |
| 1 | Decision Tree | 0.87275 |
| 2 | Random Forest | 0.934 |
| 3 | Logistic Regression | 0.9025 |
| 4 | Gradient Boosting | 0.92825 |
In [45]:
# affichage avec tri selon les performances
DFR.sort!(df_results,:accuracy,rev=true)
df_results
4×2 DataFrame
| Row | model | accuracy |
|---|---|---|
| String | Float64 | |
| 1 | Random Forest | 0.934 |
| 2 | Gradient Boosting | 0.92825 |
| 3 | Logistic Regression | 0.9025 |
| 4 | Decision Tree | 0.87275 |
In [46]:
# représentation graphique
import Plots
Plots.bar(df_results.model,df_results.accuracy,legend=false)
