Miki/TestMicrosoftML
1
0
Fork 0
Code Releases Activity

Cambiada estructura de la DB y generadas algunas pruebas

Browse Source
This commit is contained in:
Miguel Á M.S.
2023-01-20 19:09:17 +01:00
parent 72978b216c
commit 555fd0e140
2 changed files with 176 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

246
testML/S4i_SimularCruces.cs
View File

@@ -67,13 +67,13 @@ namespace testML.S4i_Simulador
}
}
//foreach(var column in modelosIA.Keys)
//{
// if(allColumns.ContainsKey(column))
// {
// allColumns.Remove(column);
// }
//}
foreach (var column in allColumns.Keys.ToArray())
{
if (!modelosIA.ContainsKey(column))
{
allColumns.Remove(column);
}
}
var allCodes = (from x in all select x.Code).Distinct().ToList();
@@ -155,36 +155,148 @@ namespace testML.S4i_Simulador
#region Creamos la tabla de la simulación
db.Execute("DROP TABLE IF EXISTS Cruce;");
sql.Clear();
sql.AppendLine(@"CREATE TABLE Cruce (Id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, Female TEXT NOT NULL, Male TEXT NOT NULL");
foreach (var col in allColumns.Values)
{
sql.Append(", ");
sql.Append(col);
sql.Append(" TEXT");
}
sql.AppendLine(");");
db.Execute(sql.ToString());
db.Execute(@"CREATE TABLE Cruce (Id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, Female TEXT NOT NULL, Male TEXT NOT NULL);");
// Le hacemos un índice
db.Execute(@"CREATE UNIQUE INDEX IX_Cruce_FemaleMale ON Cruce (
Female COLLATE BINARY ASC,
Male COLLATE BINARY ASC
db.Execute(@"CREATE UNIQUE INDEX IX_Cruce_FemaleMale ON Cruce (Female COLLATE BINARY ASC, Male COLLATE BINARY ASC);");
db.Execute("DROP TABLE IF EXISTS CruceItem;");
db.Execute(@"CREATE TABLE CruceItem (
Id INTEGER PRIMARY KEY AUTOINCREMENT
UNIQUE
NOT NULL,
IdCruce INTEGER REFERENCES Cruce (Id) ON DELETE CASCADE
ON UPDATE CASCADE
NOT NULL,
Name TEXT NOT NULL,
Value TEXT
);");
db.Execute(@"CREATE UNIQUE INDEX IX_CruceItem_IdCruce_Name ON CruceItem (
IdCruce COLLATE BINARY ASC,
Name COLLATE BINARY ASC
);");
#endregion
db.Execute("DROP VIEW IF EXISTS CruceItemAC;");
db.Execute(@"CREATE VIEW CruceItemAC AS
SELECT C.Id AS Id, C.IdCruce AS IdCruce, C.Name AS Name
, CASE C.Value
WHEN 'B' THEN 'C'
WHEN 'H' THEN 'A/C'
ELSE C.Value END AS Value
FROM CruceItem C");
db.BeginTransaction();
//ELiminamos columnas
foreach (var col in allColumns.Keys)
try
{
if (!modelosIA.ContainsKey(col))
db.Execute("DROP VIEW IF EXISTS CruceWithData;");
sql.Clear();
sql.AppendLine("CREATE VIEW CruceWithData AS");
sql.AppendLine("SELECT C.Id AS Id, C.Female AS Female, C.Male AS Male");
foreach (var col in allColumns.Values)
{
db.Execute("ALTER TABLE Cruce DROP COLUMN " + allColumns[col] + ";");
sql.Append(" , ");
sql.Append(col);
sql.Append(".Value AS ");
sql.Append(col);
sql.AppendLine();
}
sql.AppendLine(" FROM Cruce C");
foreach (var col in allColumns.Values)
{
sql.Append(" INNER JOIN CruceItem ");
sql.Append(col);
sql.Append(" ON ");
sql.Append(col);
sql.Append(".IdCruce = C.Id AND ");
sql.Append(col);
sql.Append(".Name = '");
sql.Append(col);
sql.Append("'");
sql.AppendLine();
}
db.Execute(sql.ToString());
}
db.EndTransaction(true);
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
try
{
db.Execute("DROP VIEW IF EXISTS CruceWithData_v2;");
sql.Clear();
sql.AppendLine("CREATE VIEW CruceWithData_v2 AS");
sql.AppendLine("SELECT C.Id AS Id, C.Female AS Female, C.Male AS Male");
foreach (var col in allColumns.Values)
{
sql.AppendLine(string.Format(" , (SELECT T{0}.Value FROM CruceItem T{0} WHERE T{0}.IdCruce = C.Id AND T{0}.Name = '{0}' LIMIT 1) AS {0}", col));
}
sql.AppendLine(" FROM Cruce C");
db.Execute(sql.ToString());
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
try
{
db.Execute("DROP VIEW IF EXISTS CruceWithDataAC;");
sql.Clear();
sql.AppendLine("CREATE VIEW CruceWithDataAC AS");
sql.AppendLine("SELECT C.Id AS Id, C.Female AS Female, C.Male AS Male");
foreach (var col in allColumns.Values)
{
sql.Append(" , ");
sql.Append(col);
sql.Append(".Value AS ");
sql.Append(col);
sql.AppendLine();
}
sql.AppendLine(" FROM Cruce C");
foreach (var col in allColumns.Values)
{
sql.Append(" INNER JOIN CruceItemAC ");
sql.Append(col);
sql.Append(" ON ");
sql.Append(col);
sql.Append(".IdCruce = C.Id AND ");
sql.Append(col);
sql.Append(".Name = '");
sql.Append(col);
sql.Append("'");
sql.AppendLine();
}
db.Execute(sql.ToString());
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
try
{
db.Execute("DROP VIEW IF EXISTS CruceWithDataAC_v2;");
sql.Clear();
sql.AppendLine("CREATE VIEW CruceWithDataAC_v2 AS");
sql.AppendLine("SELECT C.Id AS Id, C.Female AS Female, C.Male AS Male");
foreach (var col in allColumns.Values)
{
sql.AppendLine(string.Format(" , (SELECT T{0}.Value FROM CruceItemAC T{0} WHERE T{0}.IdCruce = C.Id AND T{0}.Name = '{0}' LIMIT 1) AS {0}", col));
}
sql.AppendLine(" FROM Cruce C");
db.Execute(sql.ToString());
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
#endregion
var limit = individuos.Length * individuos.Length;
@@ -224,24 +336,7 @@ namespace testML.S4i_Simulador
private static void SimulateCross(SQLiteConnector db, Individuo asFemale, Individuo asMale)
{
var cmd = db.GetCommand();
var sql = new StringBuilder();
sql.Append("INSERT INTO Cruce(Female, Male");
foreach (var model in modelosIA.Values)
{
sql.Append(", " + allColumns[model.Name]);
}
sql.Append(") VALUES(@pFemale, @pMale");
cmd["@pFemale"] = asFemale.Code;
cmd["@pMale"] = asMale.Code;
foreach (var model in modelosIA.Values)
{
sql.Append(", @p" + allColumns[model.Name]);
}
sql.Append(");");
// Calculamos los valores
var tasks = new Task[modelosIA.Values.Count];
var c = 0;
foreach (var model in modelosIA.Values)
@@ -251,19 +346,29 @@ namespace testML.S4i_Simulador
}
Task.WaitAll(tasks);
foreach (var model in modelosIA.Values)
// Insertamos en la DB
db.BeginTransaction();
try
{
var colDB = allColumns[model.Name];
var value = model.Prediction.GetValue("DESCENDIENTE_" + model.Name) as string;
if (value != null && value.StartsWith(colDB))
{
value = value.Substring(colDB.Length + 1);
}
db.Execute("INSERT INTO Cruce(Female, Male) VALUES({0}, {1})", asFemale.Code, asMale.Code);
var id = db.GetLastInsertId();
cmd["@p" + colDB] = value;
foreach (var model in modelosIA.Values)
{
var colDB = allColumns[model.Name];
var value = model.Prediction.GetValue("DESCENDIENTE_" + model.Name) as string;
if (value != null && value.StartsWith(colDB))
{
value = value.Substring(colDB.Length + 1);
}
db.Execute("INSERT INTO CruceItem(IdCruce, Name, Value) VALUES({0}, {1}, {2})", id, colDB, value);
}
}
finally
{
db.EndTransaction(true);
}
cmd.SQL = sql.ToString();
db.Execute(cmd);
}
@@ -290,37 +395,33 @@ namespace testML.S4i_Simulador
{
var count = 0;
var result = new Dictionary<string, ModelosIA>();
var tasks = new List<Task>();
foreach (var filename in Directory.GetFiles("Modelos", "*.zip"))
var files = Directory.GetFiles("Modelos", "*.zip").OrderBy(x => x).ToArray();
for (var c = 0; c < files.Length; c++)
{
count++;
tasks.Add(LoadModels(result, filename));
}
//if (count > 5) break;
Task.WaitAll(tasks.ToArray());
var filename = files[c];
return result;
}
private static Task LoadModels(Dictionary<string, ModelosIA> result, string filename)
{
return Task.Run(() =>
{
//if (count >= 5) break;
var name = Path.GetFileNameWithoutExtension(filename);
var matchName = columnByModelName.Match(name);
Console.WriteLine("Loading model for: " + matchName.Groups[1].Value);
Console.Write(string.Format("{0}/{1} Loading: {2}... ", c + 1, files.Length, matchName.Groups[1].Value));
var model = new ModelosIA();
model.Name = matchName.Groups[1].Value;
Console.Write("Model... ");
model.Model = mlContext.Model.Load(filename, out DataViewSchema schema);
model.Schema = schema;
var dll = Path.Combine(Path.GetDirectoryName(filename), name + ".dll");
if (File.Exists(dll))
{
Console.Write("Obj... ");
var a = Assembly.LoadFrom(dll);
var obj = "OBJ" + name.Replace(".", "_");
@@ -329,6 +430,7 @@ namespace testML.S4i_Simulador
model.Data = (IDictionaryToObjectConverter)Activator.CreateInstance(dataType);
Console.Write("Prediction engine... ");
var createPredictionEngineMethod = mlContext.Model.GetType().GetMethods().Where(x => x.Name == "CreatePredictionEngine" && x.IsGenericMethodDefinition).FirstOrDefault();
var createPredictionEngineMethodObj = createPredictionEngineMethod.MakeGenericMethod(dataType, predictionType);
model.PredictionEngine = createPredictionEngineMethodObj.Invoke(mlContext.Model, new object[] { model.Model, null, null, null });
@@ -343,12 +445,14 @@ namespace testML.S4i_Simulador
{
result.Add(model.Name, model);
}
Console.WriteLine();
}
Console.WriteLine("Model for: " + matchName.Groups[1].Value + " loaded!");
});
return result;
}
private static IEnumerable<Individuo> ObtenerIndividuos()
{
foreach (var item in sourceData)