diff --git a/testML/Program.cs b/testML/Program.cs index 2513abe..220d83a 100644 --- a/testML/Program.cs +++ b/testML/Program.cs @@ -49,7 +49,7 @@ namespace testML for (var r = headerRow.RowNum + 1; r < sheet.LastRowNum - 1; r++) { - //if (r == 100) break; + if (r == 200) break; Console.WriteLine(string.Format("{0} / {1}", r, sheet.LastRowNum - 1)); var row = sheet.GetRow(r); diff --git a/testML/S4i_SimularCruces.cs b/testML/S4i_SimularCruces.cs index b0de15c..efeeacd 100644 --- a/testML/S4i_SimularCruces.cs +++ b/testML/S4i_SimularCruces.cs @@ -67,13 +67,13 @@ namespace testML.S4i_Simulador } } - //foreach(var column in modelosIA.Keys) - //{ - // if(allColumns.ContainsKey(column)) - // { - // allColumns.Remove(column); - // } - //} + foreach (var column in allColumns.Keys.ToArray()) + { + if (!modelosIA.ContainsKey(column)) + { + allColumns.Remove(column); + } + } var allCodes = (from x in all select x.Code).Distinct().ToList(); @@ -155,36 +155,148 @@ namespace testML.S4i_Simulador #region Creamos la tabla de la simulación db.Execute("DROP TABLE IF EXISTS Cruce;"); - sql.Clear(); - sql.AppendLine(@"CREATE TABLE Cruce (Id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, Female TEXT NOT NULL, Male TEXT NOT NULL"); - - foreach (var col in allColumns.Values) - { - sql.Append(", "); - sql.Append(col); - sql.Append(" TEXT"); - } - sql.AppendLine(");"); - db.Execute(sql.ToString()); + db.Execute(@"CREATE TABLE Cruce (Id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, Female TEXT NOT NULL, Male TEXT NOT NULL);"); // Le hacemos un índice - db.Execute(@"CREATE UNIQUE INDEX IX_Cruce_FemaleMale ON Cruce ( - Female COLLATE BINARY ASC, - Male COLLATE BINARY ASC + db.Execute(@"CREATE UNIQUE INDEX IX_Cruce_FemaleMale ON Cruce (Female COLLATE BINARY ASC, Male COLLATE BINARY ASC);"); + + db.Execute("DROP TABLE IF EXISTS CruceItem;"); + db.Execute(@"CREATE TABLE CruceItem ( + Id INTEGER PRIMARY KEY AUTOINCREMENT + UNIQUE + NOT NULL, + IdCruce INTEGER REFERENCES Cruce (Id) ON DELETE CASCADE + ON UPDATE CASCADE + NOT NULL, + Name TEXT NOT NULL, + Value TEXT +);"); + db.Execute(@"CREATE UNIQUE INDEX IX_CruceItem_IdCruce_Name ON CruceItem ( + IdCruce COLLATE BINARY ASC, + Name COLLATE BINARY ASC );"); - #endregion + db.Execute("DROP VIEW IF EXISTS CruceItemAC;"); + db.Execute(@"CREATE VIEW CruceItemAC AS +SELECT C.Id AS Id, C.IdCruce AS IdCruce, C.Name AS Name + , CASE C.Value + WHEN 'B' THEN 'C' + WHEN 'H' THEN 'A/C' + ELSE C.Value END AS Value +FROM CruceItem C"); - db.BeginTransaction(); - //ELiminamos columnas - foreach (var col in allColumns.Keys) + try { - if (!modelosIA.ContainsKey(col)) + db.Execute("DROP VIEW IF EXISTS CruceWithData;"); + sql.Clear(); + sql.AppendLine("CREATE VIEW CruceWithData AS"); + sql.AppendLine("SELECT C.Id AS Id, C.Female AS Female, C.Male AS Male"); + foreach (var col in allColumns.Values) { - db.Execute("ALTER TABLE Cruce DROP COLUMN " + allColumns[col] + ";"); + sql.Append(" , "); + sql.Append(col); + sql.Append(".Value AS "); + sql.Append(col); + sql.AppendLine(); } + sql.AppendLine(" FROM Cruce C"); + + foreach (var col in allColumns.Values) + { + sql.Append(" INNER JOIN CruceItem "); + sql.Append(col); + sql.Append(" ON "); + sql.Append(col); + sql.Append(".IdCruce = C.Id AND "); + sql.Append(col); + sql.Append(".Name = '"); + sql.Append(col); + sql.Append("'"); + sql.AppendLine(); + } + db.Execute(sql.ToString()); } - db.EndTransaction(true); + catch (Exception ex) + { + Console.WriteLine(ex.ToString()); + } + + try + { + db.Execute("DROP VIEW IF EXISTS CruceWithData_v2;"); + sql.Clear(); + sql.AppendLine("CREATE VIEW CruceWithData_v2 AS"); + sql.AppendLine("SELECT C.Id AS Id, C.Female AS Female, C.Male AS Male"); + foreach (var col in allColumns.Values) + { + sql.AppendLine(string.Format(" , (SELECT T{0}.Value FROM CruceItem T{0} WHERE T{0}.IdCruce = C.Id AND T{0}.Name = '{0}' LIMIT 1) AS {0}", col)); + } + sql.AppendLine(" FROM Cruce C"); + + db.Execute(sql.ToString()); + } + catch (Exception ex) + { + Console.WriteLine(ex.ToString()); + } + + try + { + db.Execute("DROP VIEW IF EXISTS CruceWithDataAC;"); + sql.Clear(); + sql.AppendLine("CREATE VIEW CruceWithDataAC AS"); + sql.AppendLine("SELECT C.Id AS Id, C.Female AS Female, C.Male AS Male"); + foreach (var col in allColumns.Values) + { + sql.Append(" , "); + sql.Append(col); + sql.Append(".Value AS "); + sql.Append(col); + sql.AppendLine(); + } + sql.AppendLine(" FROM Cruce C"); + + foreach (var col in allColumns.Values) + { + sql.Append(" INNER JOIN CruceItemAC "); + sql.Append(col); + sql.Append(" ON "); + sql.Append(col); + sql.Append(".IdCruce = C.Id AND "); + sql.Append(col); + sql.Append(".Name = '"); + sql.Append(col); + sql.Append("'"); + sql.AppendLine(); + } + db.Execute(sql.ToString()); + } + catch (Exception ex) + { + Console.WriteLine(ex.ToString()); + } + + try + { + db.Execute("DROP VIEW IF EXISTS CruceWithDataAC_v2;"); + sql.Clear(); + sql.AppendLine("CREATE VIEW CruceWithDataAC_v2 AS"); + sql.AppendLine("SELECT C.Id AS Id, C.Female AS Female, C.Male AS Male"); + foreach (var col in allColumns.Values) + { + sql.AppendLine(string.Format(" , (SELECT T{0}.Value FROM CruceItemAC T{0} WHERE T{0}.IdCruce = C.Id AND T{0}.Name = '{0}' LIMIT 1) AS {0}", col)); + } + sql.AppendLine(" FROM Cruce C"); + + db.Execute(sql.ToString()); + } + catch (Exception ex) + { + Console.WriteLine(ex.ToString()); + } + + + #endregion var limit = individuos.Length * individuos.Length; @@ -224,24 +336,7 @@ namespace testML.S4i_Simulador private static void SimulateCross(SQLiteConnector db, Individuo asFemale, Individuo asMale) { - var cmd = db.GetCommand(); - var sql = new StringBuilder(); - sql.Append("INSERT INTO Cruce(Female, Male"); - foreach (var model in modelosIA.Values) - { - sql.Append(", " + allColumns[model.Name]); - } - sql.Append(") VALUES(@pFemale, @pMale"); - - cmd["@pFemale"] = asFemale.Code; - cmd["@pMale"] = asMale.Code; - foreach (var model in modelosIA.Values) - { - sql.Append(", @p" + allColumns[model.Name]); - } - sql.Append(");"); - - + // Calculamos los valores var tasks = new Task[modelosIA.Values.Count]; var c = 0; foreach (var model in modelosIA.Values) @@ -251,19 +346,29 @@ namespace testML.S4i_Simulador } Task.WaitAll(tasks); - foreach (var model in modelosIA.Values) + // Insertamos en la DB + db.BeginTransaction(); + try { - var colDB = allColumns[model.Name]; - var value = model.Prediction.GetValue("DESCENDIENTE_" + model.Name) as string; - if (value != null && value.StartsWith(colDB)) - { - value = value.Substring(colDB.Length + 1); - } + db.Execute("INSERT INTO Cruce(Female, Male) VALUES({0}, {1})", asFemale.Code, asMale.Code); + var id = db.GetLastInsertId(); - cmd["@p" + colDB] = value; + foreach (var model in modelosIA.Values) + { + var colDB = allColumns[model.Name]; + var value = model.Prediction.GetValue("DESCENDIENTE_" + model.Name) as string; + if (value != null && value.StartsWith(colDB)) + { + value = value.Substring(colDB.Length + 1); + } + + db.Execute("INSERT INTO CruceItem(IdCruce, Name, Value) VALUES({0}, {1}, {2})", id, colDB, value); + } + } + finally + { + db.EndTransaction(true); } - cmd.SQL = sql.ToString(); - db.Execute(cmd); } @@ -290,37 +395,33 @@ namespace testML.S4i_Simulador { var count = 0; var result = new Dictionary(); - var tasks = new List(); - foreach (var filename in Directory.GetFiles("Modelos", "*.zip")) + + var files = Directory.GetFiles("Modelos", "*.zip").OrderBy(x => x).ToArray(); + + for (var c = 0; c < files.Length; c++) { count++; - tasks.Add(LoadModels(result, filename)); - } + //if (count > 5) break; - Task.WaitAll(tasks.ToArray()); + var filename = files[c]; - return result; - } - private static Task LoadModels(Dictionary result, string filename) - { - return Task.Run(() => - { - //if (count >= 5) break; var name = Path.GetFileNameWithoutExtension(filename); var matchName = columnByModelName.Match(name); - Console.WriteLine("Loading model for: " + matchName.Groups[1].Value); + Console.Write(string.Format("{0}/{1} Loading: {2}... ", c + 1, files.Length, matchName.Groups[1].Value)); var model = new ModelosIA(); model.Name = matchName.Groups[1].Value; + Console.Write("Model... "); model.Model = mlContext.Model.Load(filename, out DataViewSchema schema); model.Schema = schema; var dll = Path.Combine(Path.GetDirectoryName(filename), name + ".dll"); if (File.Exists(dll)) { + Console.Write("Obj... "); var a = Assembly.LoadFrom(dll); var obj = "OBJ" + name.Replace(".", "_"); @@ -329,6 +430,7 @@ namespace testML.S4i_Simulador model.Data = (IDictionaryToObjectConverter)Activator.CreateInstance(dataType); + Console.Write("Prediction engine... "); var createPredictionEngineMethod = mlContext.Model.GetType().GetMethods().Where(x => x.Name == "CreatePredictionEngine" && x.IsGenericMethodDefinition).FirstOrDefault(); var createPredictionEngineMethodObj = createPredictionEngineMethod.MakeGenericMethod(dataType, predictionType); model.PredictionEngine = createPredictionEngineMethodObj.Invoke(mlContext.Model, new object[] { model.Model, null, null, null }); @@ -343,12 +445,14 @@ namespace testML.S4i_Simulador { result.Add(model.Name, model); } + Console.WriteLine(); + } - Console.WriteLine("Model for: " + matchName.Groups[1].Value + " loaded!"); - }); + return result; } + private static IEnumerable ObtenerIndividuos() { foreach (var item in sourceData)