515 lines
14 KiB
C#
515 lines
14 KiB
C#
using FFSoft.SQLiteUtilities;
|
|
using Microsoft.CodeAnalysis.CSharp.Syntax;
|
|
using Microsoft.CodeAnalysis.Operations;
|
|
using Microsoft.ML;
|
|
using NPOI.SS.Formula.Functions;
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Collections.Specialized;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Net;
|
|
using System.Reflection;
|
|
using System.Reflection.PortableExecutable;
|
|
using System.Text;
|
|
using System.Text.RegularExpressions;
|
|
using System.Threading.Tasks;
|
|
using Tensorboard;
|
|
|
|
namespace testML.S4i_Simulador
|
|
{
|
|
internal class S4i_SimularCruces
|
|
{
|
|
static Regex removeCr = new Regex(@"(.+)_CR\d+$", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
|
static Regex columnByModelName = new Regex(@"DESCENDIENTE_(.+_CR\d+)\.", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
|
|
|
static List<Dictionary<string, object>> sourceData;
|
|
static Dictionary<string, string> allColumns;
|
|
static Dictionary<string, ModelosIA> modelosIA;
|
|
static MLContext mlContext;
|
|
|
|
public static void Run(List<Dictionary<string, object>> sourceData)
|
|
{
|
|
mlContext = new MLContext();
|
|
modelosIA = LoadModels();
|
|
|
|
S4i_SimularCruces.sourceData = sourceData;
|
|
|
|
var all = (from x in ObtenerIndividuos() select x).ToList();
|
|
{
|
|
allColumns = new Dictionary<string, string>();
|
|
|
|
// Buscamos el nombre de todos los atributos
|
|
foreach (var row in all)
|
|
{
|
|
foreach (var col in row.Values.Keys)
|
|
{
|
|
if (!allColumns.ContainsKey(col))
|
|
{
|
|
var m = removeCr.Match(col);
|
|
if (!m.Success) { continue; }
|
|
|
|
allColumns.Add(col, m.Groups[1].Value);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Añadimos todos los atributos que falten
|
|
foreach (var row in all)
|
|
{
|
|
foreach (var col in allColumns.Keys)
|
|
{
|
|
if (!row.Values.ContainsKey(col))
|
|
{
|
|
row.Values.Add(col, null);
|
|
}
|
|
}
|
|
}
|
|
|
|
foreach (var column in allColumns.Keys.ToArray())
|
|
{
|
|
if (!modelosIA.ContainsKey(column))
|
|
{
|
|
allColumns.Remove(column);
|
|
}
|
|
}
|
|
|
|
var allCodes = (from x in all select x.Code).Distinct().ToList();
|
|
|
|
var individuos = (from x in allCodes select all.FirstOrDefault(y => y.Code == x)).ToArray();
|
|
all = null;
|
|
|
|
var outputDB = "Crosses.sqlite";
|
|
|
|
if (!File.Exists(outputDB))
|
|
{
|
|
SQLiteConnector.CreateDatabase(outputDB);
|
|
}
|
|
|
|
using (var db = new SQLiteConnector(outputDB))
|
|
{
|
|
db.Execute("PRAGMA journal_mode = MEMORY;");
|
|
db.Execute("PRAGMA wal_autocheckpoint = 1000;");
|
|
db.Execute("PRAGMA synchronous = OFF;");
|
|
|
|
db.Execute("VACUUM");
|
|
#region Crear tabla de los individuos
|
|
|
|
db.Execute("DROP TABLE IF EXISTS Individuo;");
|
|
var sql = new StringBuilder();
|
|
sql.AppendLine(@"CREATE TABLE Individuo (Code TEXT PRIMARY KEY UNIQUE NOT NULL");
|
|
|
|
foreach (var col in allColumns.Values)
|
|
{
|
|
sql.Append(", ");
|
|
sql.Append(col);
|
|
sql.Append(" TEXT");
|
|
}
|
|
sql.AppendLine(");");
|
|
db.Execute(sql.ToString());
|
|
|
|
db.BeginTransaction();
|
|
foreach (var i in individuos)
|
|
{
|
|
db.Execute("INSERT INTO Individuo(Code) VALUES({0})", i.Code);
|
|
|
|
|
|
var cmd = db.GetCommand();
|
|
sql.Clear();
|
|
sql.Append("UPDATE Individuo SET ");
|
|
var addComa = false;
|
|
foreach (var col in allColumns.Keys)
|
|
{
|
|
var name = allColumns[col];
|
|
|
|
if (addComa)
|
|
{
|
|
sql.Append(", ");
|
|
}
|
|
else
|
|
{
|
|
addComa = true;
|
|
}
|
|
|
|
|
|
sql.Append(name);
|
|
sql.Append(" = ");
|
|
|
|
var param = "@p" + name;
|
|
sql.Append(param);
|
|
|
|
var v = i.Values[col];
|
|
if (string.IsNullOrEmpty(v as string)) { v = null; }
|
|
|
|
cmd[param] = v?.ToString().Substring(name.Length + 1);
|
|
}
|
|
sql.Append(" WHERE Code = @pCode");
|
|
cmd["@pCode"] = i.Code;
|
|
|
|
cmd.SQL = sql.ToString();
|
|
db.Execute(cmd);
|
|
}
|
|
db.EndTransaction(true);
|
|
#endregion
|
|
|
|
|
|
#region Creamos la tabla de la simulación
|
|
|
|
db.Execute("DROP TABLE IF EXISTS Cruce;");
|
|
db.Execute(@"CREATE TABLE Cruce (Id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, Female TEXT NOT NULL, Male TEXT NOT NULL);");
|
|
|
|
// Le hacemos un índice
|
|
db.Execute(@"CREATE UNIQUE INDEX IX_Cruce_FemaleMale ON Cruce (Female COLLATE BINARY ASC, Male COLLATE BINARY ASC);");
|
|
|
|
db.Execute("DROP TABLE IF EXISTS CruceItem;");
|
|
db.Execute(@"CREATE TABLE CruceItem (
|
|
Id INTEGER PRIMARY KEY AUTOINCREMENT
|
|
UNIQUE
|
|
NOT NULL,
|
|
IdCruce INTEGER REFERENCES Cruce (Id) ON DELETE CASCADE
|
|
ON UPDATE CASCADE
|
|
NOT NULL,
|
|
Name TEXT NOT NULL,
|
|
Value TEXT
|
|
);");
|
|
db.Execute(@"CREATE UNIQUE INDEX IX_CruceItem_IdCruce_Name ON CruceItem (
|
|
IdCruce COLLATE BINARY ASC,
|
|
Name COLLATE BINARY ASC
|
|
);");
|
|
|
|
db.Execute("DROP VIEW IF EXISTS CruceItemAC;");
|
|
db.Execute(@"CREATE VIEW CruceItemAC AS
|
|
SELECT C.Id AS Id, C.IdCruce AS IdCruce, C.Name AS Name
|
|
, CASE C.Value
|
|
WHEN 'B' THEN 'C'
|
|
WHEN 'H' THEN 'A/C'
|
|
ELSE C.Value END AS Value
|
|
FROM CruceItem C");
|
|
|
|
try
|
|
{
|
|
db.Execute("DROP VIEW IF EXISTS CruceWithData;");
|
|
sql.Clear();
|
|
sql.AppendLine("CREATE VIEW CruceWithData AS");
|
|
sql.AppendLine("SELECT C.Id AS Id, C.Female AS Female, C.Male AS Male");
|
|
foreach (var col in allColumns.Values)
|
|
{
|
|
sql.Append(" , ");
|
|
sql.Append(col);
|
|
sql.Append(".Value AS ");
|
|
sql.Append(col);
|
|
sql.AppendLine();
|
|
}
|
|
sql.AppendLine(" FROM Cruce C");
|
|
|
|
foreach (var col in allColumns.Values)
|
|
{
|
|
sql.Append(" INNER JOIN CruceItem ");
|
|
sql.Append(col);
|
|
sql.Append(" ON ");
|
|
sql.Append(col);
|
|
sql.Append(".IdCruce = C.Id AND ");
|
|
sql.Append(col);
|
|
sql.Append(".Name = '");
|
|
sql.Append(col);
|
|
sql.Append("'");
|
|
sql.AppendLine();
|
|
}
|
|
db.Execute(sql.ToString());
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine(ex.ToString());
|
|
}
|
|
|
|
try
|
|
{
|
|
db.Execute("DROP VIEW IF EXISTS CruceWithData_v2;");
|
|
sql.Clear();
|
|
sql.AppendLine("CREATE VIEW CruceWithData_v2 AS");
|
|
sql.AppendLine("SELECT C.Id AS Id, C.Female AS Female, C.Male AS Male");
|
|
foreach (var col in allColumns.Values)
|
|
{
|
|
sql.AppendLine(string.Format(" , (SELECT T{0}.Value FROM CruceItem T{0} WHERE T{0}.IdCruce = C.Id AND T{0}.Name = '{0}' LIMIT 1) AS {0}", col));
|
|
}
|
|
sql.AppendLine(" FROM Cruce C");
|
|
|
|
db.Execute(sql.ToString());
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine(ex.ToString());
|
|
}
|
|
|
|
try
|
|
{
|
|
db.Execute("DROP VIEW IF EXISTS CruceWithDataAC;");
|
|
sql.Clear();
|
|
sql.AppendLine("CREATE VIEW CruceWithDataAC AS");
|
|
sql.AppendLine("SELECT C.Id AS Id, C.Female AS Female, C.Male AS Male");
|
|
foreach (var col in allColumns.Values)
|
|
{
|
|
sql.Append(" , ");
|
|
sql.Append(col);
|
|
sql.Append(".Value AS ");
|
|
sql.Append(col);
|
|
sql.AppendLine();
|
|
}
|
|
sql.AppendLine(" FROM Cruce C");
|
|
|
|
foreach (var col in allColumns.Values)
|
|
{
|
|
sql.Append(" INNER JOIN CruceItemAC ");
|
|
sql.Append(col);
|
|
sql.Append(" ON ");
|
|
sql.Append(col);
|
|
sql.Append(".IdCruce = C.Id AND ");
|
|
sql.Append(col);
|
|
sql.Append(".Name = '");
|
|
sql.Append(col);
|
|
sql.Append("'");
|
|
sql.AppendLine();
|
|
}
|
|
db.Execute(sql.ToString());
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine(ex.ToString());
|
|
}
|
|
|
|
try
|
|
{
|
|
db.Execute("DROP VIEW IF EXISTS CruceWithDataAC_v2;");
|
|
sql.Clear();
|
|
sql.AppendLine("CREATE VIEW CruceWithDataAC_v2 AS");
|
|
sql.AppendLine("SELECT C.Id AS Id, C.Female AS Female, C.Male AS Male");
|
|
foreach (var col in allColumns.Values)
|
|
{
|
|
sql.AppendLine(string.Format(" , (SELECT T{0}.Value FROM CruceItemAC T{0} WHERE T{0}.IdCruce = C.Id AND T{0}.Name = '{0}' LIMIT 1) AS {0}", col));
|
|
}
|
|
sql.AppendLine(" FROM Cruce C");
|
|
|
|
db.Execute(sql.ToString());
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine(ex.ToString());
|
|
}
|
|
|
|
|
|
#endregion
|
|
|
|
Console.WriteLine("Ready");
|
|
Console.ReadLine();
|
|
|
|
var limit = individuos.Length * individuos.Length;
|
|
var current = 0;
|
|
foreach (var asFemale in individuos)
|
|
{
|
|
FillModel("FEMENINO_", asFemale);
|
|
|
|
foreach (var asMale in individuos)
|
|
{
|
|
current++;
|
|
|
|
if (asFemale.Code == asMale.Code) { continue; } // Ignoramos las autofecundaciones
|
|
|
|
// Si existe el curce en un sentido o en el otro lo ignoramos
|
|
if (db.Exists("SELECT * FROM Cruce WHERE (Female = {0} AND Male = {1}) OR (Female = {1} AND Male = {0})", asFemale.Code, asMale.Code))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
FillModel("MASCULINO_", asMale);
|
|
|
|
Console.WriteLine("{2}/{3} {4}%: Simulating {0} x {1}",
|
|
asFemale.Code,
|
|
asMale.Code,
|
|
current.ToString("#,##0"),
|
|
limit.ToString("#,##0"),
|
|
((current / (double)limit) * 100.0).ToString("#,##0.00"));
|
|
SimulateCross(db, asFemale, asMale);
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
private static void SimulateCross(SQLiteConnector db, Individuo asFemale, Individuo asMale)
|
|
{
|
|
// Calculamos los valores
|
|
var tasks = new Task[modelosIA.Values.Count];
|
|
var c = 0;
|
|
foreach (var model in modelosIA.Values)
|
|
{
|
|
tasks[c] = MakePrediction(model);
|
|
c++;
|
|
}
|
|
Task.WaitAll(tasks);
|
|
|
|
// Insertamos en la DB
|
|
db.BeginTransaction();
|
|
try
|
|
{
|
|
db.Execute("INSERT INTO Cruce(Female, Male) VALUES({0}, {1})", asFemale.Code, asMale.Code);
|
|
var id = db.GetLastInsertId();
|
|
|
|
foreach (var model in modelosIA.Values)
|
|
{
|
|
var colDB = allColumns[model.Name];
|
|
var value = model.Prediction.GetValue("DESCENDIENTE_" + model.Name) as string;
|
|
if (value != null && value.StartsWith(colDB))
|
|
{
|
|
value = value.Substring(colDB.Length + 1);
|
|
}
|
|
|
|
db.Execute("INSERT INTO CruceItem(IdCruce, Name, Value) VALUES({0}, {1}, {2})", id, colDB, value);
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
db.EndTransaction(true);
|
|
}
|
|
|
|
}
|
|
|
|
private static Task MakePrediction(ModelosIA model)
|
|
{
|
|
return Task.Run(() =>
|
|
{
|
|
model.Prediction = (IDictionaryToObjectConverter)model.PredictionEngineMethod.Invoke(model.PredictionEngine, model.PredictionEngineMethodParameter);
|
|
});
|
|
}
|
|
|
|
private static void FillModel(string prefix, Individuo individuo)
|
|
{
|
|
foreach (var model in modelosIA.Values)
|
|
{
|
|
foreach (var col in allColumns.Keys)
|
|
{
|
|
model.Data.SetValue(prefix + col, individuo.Values[col]);
|
|
}
|
|
}
|
|
}
|
|
|
|
private static Dictionary<string, ModelosIA> LoadModels()
|
|
{
|
|
var count = 0;
|
|
var result = new Dictionary<string, ModelosIA>();
|
|
|
|
var files = Directory.GetFiles("Modelos", "*.zip").OrderBy(x => x).ToArray();
|
|
|
|
for (var c = 0; c < files.Length; c++)
|
|
{
|
|
var filename = files[c];
|
|
|
|
var name = Path.GetFileNameWithoutExtension(filename);
|
|
|
|
var matchName = columnByModelName.Match(name);
|
|
if(!matchName.Success )
|
|
{
|
|
continue;
|
|
}
|
|
|
|
count++;
|
|
if (count > 10) break;
|
|
|
|
Console.Write(string.Format("{0}/{1} Loading: {2}... ", c + 1, files.Length, matchName.Groups[1].Value));
|
|
|
|
var model = new ModelosIA();
|
|
model.Name = matchName.Groups[1].Value;
|
|
Console.Write("Model... ");
|
|
model.Model = mlContext.Model.Load(filename, out DataViewSchema schema);
|
|
model.Schema = schema;
|
|
|
|
var dll = Path.Combine(Path.GetDirectoryName(filename), name + ".dll");
|
|
if (File.Exists(dll))
|
|
{
|
|
Console.Write("Obj... ");
|
|
var a = Assembly.LoadFrom(dll);
|
|
var obj = "OBJ" + name.Replace(".", "_");
|
|
|
|
var dataType = a.GetType("DictionaryToObjectConverterNamespace." + obj);
|
|
var predictionType = a.GetType("DictionaryToObjectConverterNamespace." + obj + "Prediction");
|
|
|
|
model.Data = (IDictionaryToObjectConverter)Activator.CreateInstance(dataType);
|
|
|
|
Console.Write("Prediction engine... ");
|
|
var createPredictionEngineMethod = mlContext.Model.GetType().GetMethods().Where(x => x.Name == "CreatePredictionEngine" && x.IsGenericMethodDefinition).FirstOrDefault();
|
|
var createPredictionEngineMethodObj = createPredictionEngineMethod.MakeGenericMethod(dataType, predictionType);
|
|
model.PredictionEngine = createPredictionEngineMethodObj.Invoke(mlContext.Model, new object[] { model.Model, null, null, null });
|
|
|
|
//Test
|
|
model.PredictionEngineMethod = model.PredictionEngine.GetType().GetMethods().Where(x => x.Name == "Predict" && x.GetParameters().Length == 1 && x.GetParameters()[0].ParameterType == dataType).FirstOrDefault();
|
|
model.PredictionEngineMethodParameter = new object[] { model.Data };
|
|
|
|
}
|
|
|
|
lock (result)
|
|
{
|
|
result.Add(model.Name, model);
|
|
}
|
|
Console.WriteLine();
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
|
|
private static IEnumerable<Individuo> ObtenerIndividuos()
|
|
{
|
|
foreach (var item in sourceData)
|
|
{
|
|
yield return ObtenerIndividuo(item, "PMASCULINO", "MASCULINO_");
|
|
yield return ObtenerIndividuo(item, "PFEMENINO", "FEMENINO_");
|
|
yield return ObtenerIndividuo(item, "DESCENDIENTE", "DESCENDIENTE_");
|
|
}
|
|
}
|
|
|
|
private static Individuo ObtenerIndividuo(Dictionary<string, object> item, string name, string dataPrefix)
|
|
{
|
|
var result = new Individuo()
|
|
{
|
|
Code = item[name].ToString().Trim(),
|
|
Values = new Dictionary<string, object>()
|
|
|
|
};
|
|
|
|
foreach (var key in item.Keys)
|
|
{
|
|
if (key.StartsWith(dataPrefix))
|
|
{
|
|
result.Values.Add(key.Substring(dataPrefix.Length), item[key]);
|
|
}
|
|
}
|
|
|
|
return result;
|
|
}
|
|
}
|
|
|
|
public class Individuo
|
|
{
|
|
public string Code { get; set; }
|
|
|
|
public Dictionary<string, object> Values { get; set; }
|
|
}
|
|
|
|
public class ModelosIA
|
|
{
|
|
public string Name { get; set; }
|
|
|
|
public ITransformer Model { get; set; }
|
|
public DataViewSchema Schema { get; set; }
|
|
public IDictionaryToObjectConverter Data { get; set; }
|
|
public IDictionaryToObjectConverter Prediction { get; set; }
|
|
|
|
public object PredictionEngine { get; set; }
|
|
public MethodInfo PredictionEngineMethod { get; set; }
|
|
public object[] PredictionEngineMethodParameter { get; set; }
|
|
}
|
|
|
|
}
|