From c200f28c3a4e9542285f3c77c72e834e811db2f5 Mon Sep 17 00:00:00 2001 From: dvitel Date: Fri, 3 Feb 2023 16:56:37 -0500 Subject: [PATCH 1/3] prefer child on non-pareto --- evopie/pphc_quiz_model.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/evopie/pphc_quiz_model.py b/evopie/pphc_quiz_model.py index 4b1a383..5cb707a 100644 --- a/evopie/pphc_quiz_model.py +++ b/evopie/pphc_quiz_model.py @@ -49,15 +49,37 @@ def mutate_population(self): def compare(self, coeval_group_id: str, coeval_group: CoevaluationGroup): ''' Implements Pareto comparison ''' if len(coeval_group.objs) >= self.pareto_n: - genotype_evaluations = {genotype_id: evals - for genotype_id, evals in self.archive.at(coeval_group.inds, coeval_group.objs)} + genotype_evaluations = {genotype_id: evals for genotype_id, evals in self.archive.at(coeval_group.inds, coeval_group.objs)} parent = coeval_group.inds[0] parent_genotype_evaluation = genotype_evaluations[parent] # possible_winners = [ child for child in eval_group.inds[1:] # if (genotype_evaluations[child] == parent_genotype_evaluation).all() or not((genotype_evaluations[child] <= parent_genotype_evaluation).all()) ] #Pareto check + def child_is_better(parent_genotype_evaluation, child_genotype_evaluation): + child_domination = child_genotype_evaluation >= parent_genotype_evaluation + parent_domination = child_genotype_evaluation <= parent_genotype_evaluation + if (parent_genotype_evaluation == child_genotype_evaluation).all(): #Pareto check + #non-domination and same outcome for given students + #prefer child for diversity - other statistics is necessary + return True + elif child_domination.all(): + return True + elif parent_domination.all(): + return False + else: #non-pareto-comparable + #NOTE: if we use aggregation here - it will be same to non-Pareto comparison + return True #pick child + # child_sum = (child_genotype_evaluation.sum(), child_domination.sum()) + # parent_sum = (parent_genotype_evaluation.sum(), parent_domination.sum()) + # if child_sum == parent_sum: + # #prefer child for diversity but better to check other stats + # return True + # elif child_sum > parent_sum: + # return True + # else: #child_sum < parent_sum + # return False possible_winners = [ child for child in coeval_group.inds[1:] - if not((genotype_evaluations[child] == parent_genotype_evaluation).all()) and (genotype_evaluations[child] >= parent_genotype_evaluation).all() ] #Pareto check + if child_is_better(parent_genotype_evaluation, genotype_evaluations[child]) ] # possible_winners = [ child for child in eval_group.inds[1:] # if (genotype_evaluations[child] >= parent_genotype_evaluation).all() ] #Pareto check From bd4a22cc81bf3fcb93eaebc8a54fadf84cccfa0a Mon Sep 17 00:00:00 2001 From: dvitel Date: Sat, 11 Feb 2023 22:18:05 -0500 Subject: [PATCH 2/3] fix of quiz model builder --- .vscode/launch.json | 56 +++++++++++++++++++++++- evopie/cli.py | 14 +++--- evopie/pphc_quiz_model.py | 50 ++++++++++------------ evopie/quiz_model.py | 90 +++++++++++++++++++-------------------- evopie/rand_quiz_model.py | 27 +++++------- experiment.sh | 17 ++++++-- slurm/exp.sh | 2 +- 7 files changed, 155 insertions(+), 101 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 5550c91..fd35d53 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -155,7 +155,7 @@ "run", "-q", "1", "-s", "STEP1", - "--algo", "evopie.pphc_quiz_model.PphcQuizModelBuilder", "--algo-params", "{ \"seed\": 595, \"pop_size\": 1, \"pareto_n\": 2, \"child_n\": 1, \"gene_size\": 3}", + "--algo", "evopie.pphc_quiz_model.PphcQuizModel", "--algo-params", "{ \"seed\": 595, \"pop_size\": 1, \"pareto_n\": 2, \"child_n\": 1, \"gene_size\": 3}", "--evo-output", "algo/pphc-1-2-1-3.json", "--archive-output", "algo/a.csv", "--random-seed", "17" ] @@ -261,7 +261,59 @@ "--fig-name", "--" ], "console": "internalConsole" - } + }, + { + "name": "CLI: quiz deca-experiment", + "type": "python", + "request": "launch", + "module": "flask", + "env" : { + "FLASK_APP" : "app.py", + "FLASK_ENV" : "development", + "FLASK_DEBUG" : "1" + }, + "args" : [ + "quiz", "deca-experiment", + "--deca-input", "deca-spaces/space-1_1_1-s_0-2.json", + "--num-runs", "2", + "--algo", "{ \"id\": \"pphc-1-2-1-3\", \"algo\":\"evopie.pphc_quiz_model.PphcQuizModel\", \"pop_size\": 1, \"pareto_n\": 2, \"child_n\": 1, \"gene_size\": 3}" + ], + "console": "internalConsole" + }, + { + "name": "CLI: quiz deca-experiment RAND", + "type": "python", + "request": "launch", + "module": "flask", + "env" : { + "FLASK_APP" : "app.py", + "FLASK_ENV" : "development", + "FLASK_DEBUG" : "1" + }, + "args" : [ + "quiz", "deca-experiment", + "--deca-input", "deca-spaces/space-1_1_1-s_0-2.json", + "--num-runs", "2", + "--algo", "{ \"id\": \"rand-3\", \"algo\":\"evopie.rand_quiz_model.RandomQuizModel\", \"n\": 3, \"seed\": 313}" + ], + "console": "internalConsole" + }, + { + "name": "CLI: quiz export", + "type": "python", + "request": "launch", + "module": "flask", + "env" : { + "FLASK_APP" : "app.py", + "FLASK_ENV" : "development", + "FLASK_DEBUG" : "1" + }, + "args" : [ + "quiz", "export", + "-q", "1" + ], + "console": "internalConsole" + } ] } diff --git a/evopie/cli.py b/evopie/cli.py index fc69902..2b2603c 100644 --- a/evopie/cli.py +++ b/evopie/cli.py @@ -15,7 +15,7 @@ from evopie import APP, deca, models from evopie.config import QUIZ_ATTEMPT_STEP1, QUIZ_STEP1, QUIZ_STEP2, ROLE_STUDENT from evopie.utils import groupby, unpack_key -from evopie.quiz_model import get_quiz_builder, set_default_builder +from evopie.quiz_model import get_quiz_builder, set_quiz_model def throw_on_http_fail(resp: TestResponse, status: int = 400): if resp.status_code >= status: @@ -476,9 +476,9 @@ def export_student_knowledge(output): def simulate_quiz(quiz, instructor, password, no_algo, algo, algo_params, rnd, n_times, archive_output, evo_output, step, knowledge_selection, likes, justify_response, email_format, random_seed): rnd_state = np.random.RandomState(random_seed) if no_algo: - set_default_builder(None) + set_quiz_model(None) elif algo is not None: - set_default_builder(algo, settings = json.loads(algo_params) if algo_params is not None else {}) + set_quiz_model(algo, settings = json.loads(algo_params) if algo_params is not None else {}) def simulate_step(step): with APP.app_context(): @@ -584,8 +584,8 @@ def simulate_step(step): sys.stdout.write(f"EVO algo: {quiz_model.__class__}\nEVO settings: {model_state}\n") if evo_output: with open(evo_output, 'w') as evo_output_json_file: - evo_output_json_file.write(json.dumps({"algo": quiz_model.__class__.__name__, - "quiz_model":model_state, "explored_search_space_size": explored_search_space_size, + evo_output_json_file.write(json.dumps({**model_state, "algo": quiz_model.__class__.__name__, + "explored_search_space_size": explored_search_space_size, "search_space_size": search_space_size}, indent=4)) sys.stdout.write(f"[{run_idx + 1}/{n_times}] Step1 quiz {quiz} finished\n{quiz_model.to_dataframe()}\n") if QUIZ_STEP2 in step: @@ -607,7 +607,7 @@ def calc_deca_metrics(algo_input, deca_space, params, input_output): algo_results = json.loads("\n".join(f.readlines())) with open(deca_space, 'r') as f: space = deca.load_space_from_json("\n".join(f.readlines())) - population_distractors = algo_results["population_distractors"] + population_distractors = algo_results["distractors"] metrics_map = {"algo":algo_input,"deca": deca_space, **{p:algo_results.get(p, np.nan) for p in params}, **deca.dimension_coverage(space, population_distractors), **deca.avg_rank_of_repr(space, population_distractors), @@ -731,6 +731,8 @@ def init_experiment(num_questions, num_distractors, num_students, axes_number, a def run_experiment(deca_input, algo, algo_folder, random_seed, results_folder, num_runs): runner = APP.test_cli_runner() res = runner.invoke(args=["student", "knows", "-kr", "--deca-input", deca_input ]) + if res.exit_code != 0: + print(res.stdout) assert res.exit_code == 0 os.makedirs(algo_folder, exist_ok=True) os.makedirs(results_folder, exist_ok=True) diff --git a/evopie/pphc_quiz_model.py b/evopie/pphc_quiz_model.py index 5cb707a..6c6c12e 100644 --- a/evopie/pphc_quiz_model.py +++ b/evopie/pphc_quiz_model.py @@ -1,15 +1,13 @@ ''' implementation of pphc quiz model ''' -from datetime import datetime from math import comb, prod import numpy as np -from typing import Any, Optional -from pandas import DataFrame +from typing import Any from evopie import APP, models from evopie.utils import groupby from dataclasses import dataclass from numpy import unique -from evopie.quiz_model import QuizModel, QuizModelBuilder, GeneBasedUpdateMixin +from evopie.quiz_model import QuizModel, GeneBasedUpdateMixin @dataclass class CoevaluationGroup: @@ -20,19 +18,22 @@ class CoevaluationGroup: class PphcQuizModel(QuizModel, GeneBasedUpdateMixin): ''' Base class for evolution process (but still of specific form)''' + default_settings = { "pop_size": 1, "pareto_n": 2, "child_n": 1, "gene_size": 3} + def __init__(self, quiz_id: int, process: models.EvoProcess, distractors_per_question: 'dict[int, list[int]]'): super(PphcQuizModel, self).__init__(quiz_id, process, distractors_per_question) - self.gen: int = process.impl_state.get("gen", 0) - self.rnd = np.random.RandomState(process.impl_state.get("seed", None)) - self.seed = int(self.rnd.get_state()[1][0]) - self.pop_size: int = process.impl_state.get("pop_size", 1) - self.pareto_n: int = process.impl_state.get("pareto_n", 2) - self.child_n: int = process.impl_state.get("child_n", 1) - self.gene_size: int = process.impl_state.get("gene_size", 3) + settings = {**PphcQuizModel.default_settings, **process.impl_state} + self.gen: int = settings.get("gen", 0) + self.seed = settings.get("seed", None) + self.rnd = np.random.RandomState(self.seed) + self.pop_size: int = settings.get("pop_size", 1) + self.pareto_n: int = settings.get("pareto_n", 2) + self.child_n: int = settings.get("child_n", 1) + self.gene_size: int = settings.get("gene_size", 3) self.coevaluation_groups: dict[str, CoevaluationGroup] = { cgid:CoevaluationGroup(g["inds"], g["objs"], g['ppos']) - for cgid, g in process.impl_state.get("coevaluation_groups", {}).items()} + for cgid, g in settings.get("coevaluation_groups", {}).items()} self.evaluator_coevaluation_groups: dict[int, str] = { int(evaluator_id):str(group_id) - for evaluator_id, group_id in process.impl_state.get("evaluator_coevaluation_groups", {}).items() } + for evaluator_id, group_id in settings.get("evaluator_coevaluation_groups", {}).items() } #add to population new inds till moment of pop_size #noop if already inited while len(self.population) < self.pop_size: @@ -50,25 +51,28 @@ def compare(self, coeval_group_id: str, coeval_group: CoevaluationGroup): ''' Implements Pareto comparison ''' if len(coeval_group.objs) >= self.pareto_n: genotype_evaluations = {genotype_id: evals for genotype_id, evals in self.archive.at(coeval_group.inds, coeval_group.objs)} + genotype_evaluation_count = {genotype_id: num_evals for genotype_id, num_evals in self.archive.num_interactions(coeval_group.inds)} parent = coeval_group.inds[0] parent_genotype_evaluation = genotype_evaluations[parent] + parent_genotype_evaluation_count = genotype_evaluation_count[parent] # possible_winners = [ child for child in eval_group.inds[1:] # if (genotype_evaluations[child] == parent_genotype_evaluation).all() or not((genotype_evaluations[child] <= parent_genotype_evaluation).all()) ] #Pareto check - def child_is_better(parent_genotype_evaluation, child_genotype_evaluation): + def child_is_better(parent_genotype_evaluation, parent_genotype_evaluation_count, + child_genotype_evaluation, child_genotype_evaluation_count): child_domination = child_genotype_evaluation >= parent_genotype_evaluation parent_domination = child_genotype_evaluation <= parent_genotype_evaluation if (parent_genotype_evaluation == child_genotype_evaluation).all(): #Pareto check #non-domination and same outcome for given students #prefer child for diversity - other statistics is necessary - return True + return child_genotype_evaluation_count <= parent_genotype_evaluation_count #pick child if less evaluated elif child_domination.all(): return True elif parent_domination.all(): return False else: #non-pareto-comparable #NOTE: if we use aggregation here - it will be same to non-Pareto comparison - return True #pick child + return child_genotype_evaluation_count <= parent_genotype_evaluation_count #pick child if less evaluated # child_sum = (child_genotype_evaluation.sum(), child_domination.sum()) # parent_sum = (parent_genotype_evaluation.sum(), parent_domination.sum()) # if child_sum == parent_sum: @@ -79,7 +83,8 @@ def child_is_better(parent_genotype_evaluation, child_genotype_evaluation): # else: #child_sum < parent_sum # return False possible_winners = [ child for child in coeval_group.inds[1:] - if child_is_better(parent_genotype_evaluation, genotype_evaluations[child]) ] + if child_is_better(parent_genotype_evaluation, parent_genotype_evaluation_count, + genotype_evaluations[child], genotype_evaluation_count[child]) ] # possible_winners = [ child for child in eval_group.inds[1:] # if (genotype_evaluations[child] >= parent_genotype_evaluation).all() ] #Pareto check @@ -194,13 +199,4 @@ def get_internal_model(self): "gene_size": self.gene_size, "pareto_n": self.pareto_n, "child_n": self.child_n, "coevaluation_groups": {id: {"inds": g.inds, "objs": g.objs, "ppos": g.ppos} for id, g in self.coevaluation_groups.items()}, "evaluator_coevaluation_groups": self.evaluator_coevaluation_groups} - return {"population": population, "distractors": distractors, "settings": settings} - -class PphcQuizModelBuilder(QuizModelBuilder): - def __init__(self, **kwargs) -> None: - self.default_settings = { "pop_size": 1, "pareto_n": 2, "child_n": 1, "gene_size": 3} - self.settings = {**self.default_settings, **kwargs} - def get_settings(self): - return self.settings - def get_quiz_model_class(self): - return PphcQuizModel \ No newline at end of file + return {"population": population, "distractors": distractors, "settings": settings} \ No newline at end of file diff --git a/evopie/quiz_model.py b/evopie/quiz_model.py index e353002..bc463cd 100644 --- a/evopie/quiz_model.py +++ b/evopie/quiz_model.py @@ -48,7 +48,10 @@ def get(self, genotype_id: int) -> Any: return self.archive.loc[genotype_id, 'g'].g def at(self, ids, objs): - return self.archive.loc[ids, objs].iterrows() + return self.archive.loc[ids, objs].iterrows() + + def num_interactions(self, ids): + return self.archive.loc[ids, self.archive.columns != 'g'].notna().astype(int).sum(axis=1).iteritems() def set_interraction(self, id, obj, score): self.archive.loc[id, obj] = score @@ -85,7 +88,7 @@ def get_search_space_size(self) -> int: return 0 def get_internal_model(self) -> Any: #depends on impl, returns population for Evo models ''' returns anything related to state of quiz model ''' - return None + return {} @abstractmethod def prepare_for_evaluation(self, evaluator_id: int) -> 'list[tuple[int, list[int]]]': return [] @@ -107,7 +110,7 @@ def evaluate(self, evaluator_id: int, result: Any) -> None: def save(self) -> None: ''' preserve state in persistent storage''' impl_state = self.get_internal_model().get("settings", {}) - self.process.impl = self.__class__.__name__ + self.process.impl = self.__class__ .__module__ + '.' + self.__class__.__name__ self.process.impl_state = impl_state self.process.population = self.population self.process.objectives = self.objectives @@ -121,14 +124,22 @@ def to_csv(self, file_name) -> None: def to_dataframe(self) -> Optional[DataFrame]: return self.archive.archive +class GeneBasedUpdateMixin(): + def update_fitness(self: QuizModel, ind: int, evaluator_id: int, result) -> None: + # cur_score = self.archive.loc[ind, evaluator_id] + cur_score = 0 #if cur_score is None or math.isnan(cur_score) else cur_score + ind_genotype = self.archive.get(ind) + for qid, genes in ind_genotype: #result - dict where for each question we provide some data + if qid in result: + for deception in genes: + if deception == result[qid]: + cur_score += 1 + self.archive.set_interraction(ind, evaluator_id, cur_score) +from evopie.pphc_quiz_model import PphcQuizModel class QuizModelBuilder(): - def __init__(self, **kwargs) -> None: - pass - def get_settings(self) -> 'dict': - return {} - def get_quiz_model_class(self): - return None + default_quiz_model_class = PphcQuizModel + default_settings = {} def get_quiz(self, quiz_or_id: 'models.Quiz | int') -> models.Quiz: if type(quiz_or_id) == int: @@ -145,15 +156,23 @@ def get_distractor_pools(self, quiz_or_id): distractor_map = { q_id: [ d.id for d in ds ] for q_id, ds in groupby(distractors, key = lambda d: d.question_id) } return {q.id: distractor_map.get(q.question_id, []) for q in questions } + def parse_model_class(self, algo): + ''' Parse module.class to class function constructor''' + if algo is not None: + *module, builder = algo.split('.') + m = __import__(".".join(module), fromlist=[builder]) + quiz_model_class = getattr(m, builder) + else: + quiz_model_class = None + return quiz_model_class + def load_quiz_model(self, quiz_or_id, create_if_not_exist = False) -> Optional[QuizModel]: ''' Restores evo processes from db by quiz id ''' - quiz_model_class = self.get_quiz_model_class() - if quiz_model_class is None: - return None quiz = self.get_quiz(quiz_or_id) process = models.EvoProcess.query.where(models.EvoProcess.quiz_id == quiz.id, models.EvoProcess.status == QuizModelStatus.ACTIVE.value).one_or_none() if process is not None: distractors_per_question = self.get_distractor_pools(quiz) + quiz_model_class = self.parse_model_class(process.impl) quiz_model = quiz_model_class(quiz.id, process, distractors_per_question) elif create_if_not_exist: quiz_model = self.create_quiz_model(quiz) @@ -162,14 +181,18 @@ def load_quiz_model(self, quiz_or_id, create_if_not_exist = False) -> Optional[Q return quiz_model def create_quiz_model(self, quiz_or_id) -> Optional[QuizModel]: - quiz_model_class = self.get_quiz_model_class() - if quiz_model_class is None: - return None + if QuizModelBuilder.default_quiz_model_class is None: + return None + elif type(QuizModelBuilder.default_quiz_model_class) == str: + quiz_model_class = self.parse_model_class(QuizModelBuilder.default_quiz_model_class) + else: + quiz_model_class = QuizModelBuilder.default_quiz_model_class quiz = self.get_quiz(quiz_or_id) process = models.EvoProcess(quiz_id = quiz.id, start_timestamp = datetime.now(), status = QuizModelStatus.ACTIVE.value, - impl = quiz_model_class.__name__, impl_state = {**self.get_settings()}, + impl = quiz_model_class.__module__ + '.' + quiz_model_class.__name__, + impl_state = {**QuizModelBuilder.default_settings}, population = [], objectives = [], archive = []) models.DB.session.add(process) models.DB.session.commit() @@ -177,33 +200,10 @@ def create_quiz_model(self, quiz_or_id) -> Optional[QuizModel]: quiz_model = quiz_model_class(quiz.id, process, distractors_per_question) return quiz_model -class GeneBasedUpdateMixin(): - def update_fitness(self: QuizModel, ind: int, evaluator_id: int, result) -> None: - # cur_score = self.archive.loc[ind, evaluator_id] - cur_score = 0 #if cur_score is None or math.isnan(cur_score) else cur_score - ind_genotype = self.archive.get(ind) - for qid, genes in ind_genotype: #result - dict where for each question we provide some data - if qid in result: - for deception in genes: - if deception == result[qid]: - cur_score += 1 - self.archive.set_interraction(ind, evaluator_id, cur_score) +def get_quiz_builder(): + return QuizModelBuilder() -from evopie.pphc_quiz_model import PphcQuizModelBuilder -default_builder = PphcQuizModelBuilder #this is shared state but check comment for set_default_builder -default_builder_settings = {} -def set_default_builder(algo, settings = {}): - ''' DO NOT USE THIS IN PROD (where num of workers > 1). Workers will not share the state. - The functions exists for cli commands for experimentation with different algos. - ''' - global default_builder, default_builder_settings - if algo is not None: - *module, builder = algo.split('.') - m = __import__(".".join(module), fromlist=[builder]) - default_builder = getattr(m, builder) - else: - default_builder = None - default_builder_settings = settings - -def get_quiz_builder() -> QuizModelBuilder: - return (default_builder or QuizModelBuilder)(**default_builder_settings) \ No newline at end of file +def set_quiz_model(quiz_model_class, settings = {}): + ''' NOTE: this is state - used only in cli experiments ''' + QuizModelBuilder.default_quiz_model_class = quiz_model_class + QuizModelBuilder.default_settings = settings \ No newline at end of file diff --git a/evopie/rand_quiz_model.py b/evopie/rand_quiz_model.py index 05066a2..40b00a8 100644 --- a/evopie/rand_quiz_model.py +++ b/evopie/rand_quiz_model.py @@ -1,17 +1,19 @@ from math import comb, prod -from typing import Any, Optional +from typing import Any from evopie import models import numpy as np -from evopie.quiz_model import QuizModel, QuizModelBuilder, GeneBasedUpdateMixin +from evopie.quiz_model import QuizModel, GeneBasedUpdateMixin class RandomQuizModel(QuizModel, GeneBasedUpdateMixin): ''' Implements dummy choice of quiz questions - random questions at start ''' + default_settings = { "n": 3 } def __init__(self, quiz_id: int, process: models.EvoProcess, distractors_per_question: 'dict[int, list[int]]'): super(RandomQuizModel, self).__init__(quiz_id, process, distractors_per_question) - self.n = process.impl_state.get("n", 3) - self.rnd = np.random.RandomState(process.impl_state.get("seed", None)) - self.seed = int(self.rnd.get_state()[1][0]) + settings = {**RandomQuizModel.default_settings, **process.impl_state} + self.n = settings.get("n", 3) + self.seed = settings.get("seed", None) + self.rnd = np.random.RandomState(self.seed) if len(self.population) > 0: self.quiz_model = self.archive.get(self.population[0]) else: @@ -23,20 +25,13 @@ def get_search_space_size(self) -> int: return prod([ comb(len(dids), self.n) for _, dids in self.distractors_per_question.items()]) def get_internal_model(self) -> Any: #depends on impl, returns population for Evo models - return {'n':self.n, 'seed': self.rnd.get_state()[1][0]} + population = [self.archive.get(genotype_id) for genotype_id in self.population] + distractors = [d for genotype in population for (_, dids) in genotype for d in dids ] + return {"settings": {'n':self.n, 'seed': self.seed}, "population": population, "distractors": distractors} def prepare_for_evaluation(self, evaluator_id: int) -> 'list[tuple[int, list[int]]]': return [(q, list(ds)) for [q, ds] in self.quiz_model] def evaluate_internal(self, evaluator_id: int, result: Any) -> None: ''' stores results in model state ''' - self.update_fitness(evaluator_id, result) - -class RandomQuizModelBuilder(QuizModelBuilder): - def __init__(self, **kwargs) -> None: - self.default_settings = { "n": 3 } - self.settings = {**self.default_settings, **kwargs} - def get_settings(self): - return self.settings - def get_quiz_model_class(self): - return RandomQuizModel \ No newline at end of file + self.update_fitness(self.population[0], evaluator_id, result) \ No newline at end of file diff --git a/experiment.sh b/experiment.sh index 4cc531c..61bd9d5 100755 --- a/experiment.sh +++ b/experiment.sh @@ -25,16 +25,25 @@ flask deca init-many -ns 100 -nq 4 -nd 25 \ flask quiz deca-experiments \ --random-seed 23 --num-runs 3 \ - --algo '{ "id": "pphc-1-2-1-3", "algo":"evopie.pphc_quiz_model.PphcQuizModelBuilder", "pop_size": 1, "pareto_n": 2, "child_n": 1, "gene_size": 3}' + --algo '{ "id": "pphc-1-2-1-3", "algo":"evopie.pphc_quiz_model.PphcQuizModel", "pop_size": 1, "pareto_n": 2, "child_n": 1, "gene_size": 3}' flask quiz deca-experiment --deca-input deca-spaces/space-1_1_1_1_1_1_1_1-s_20-3.json \ --num-runs 3 \ - --algo '{ "id": "pphc-1-2-1-3", "algo":"evopie.pphc_quiz_model.PphcQuizModelBuilder", "pop_size": 1, "pareto_n": 2, "child_n": 1, "gene_size": 3}' + --algo '{ "id": "pphc-1-2-1-3", "algo":"evopie.pphc_quiz_model.PphcQuizModel", "pop_size": 1, "pareto_n": 2, "child_n": 1, "gene_size": 3}' -flask student knows --deca-input deca-spaces/space-1_1_1_1_1_1_1_1-s_0-1.json \ +flask quiz deca-experiment --deca-input deca-spaces/space-1_1_1-s_0-2.json \ + --num-runs 2 \ + --algo '{ "id": "pphc-1-2-1-3", "algo":"evopie.pphc_quiz_model.PphcQuizModel", "pop_size": 1, "pareto_n": 2, "child_n": 1, "gene_size": 3}' + +flask quiz deca-experiment --deca-input deca-spaces/space-1_1_1-s_0-2.json \ + --num-runs 2 \ + --algo '{ "id": "rand-3", "algo":"evopie.rand_quiz_model.RandomQuizModel", "n": 3, "seed": 313}' + + +flask student knows -kr --deca-input deca-spaces/space-1_1_1-s_0-0.json \ -o testing/students.csv -# flask quiz run -q 1 -s STEP1 --algo 'evopie.pphc_quiz_model.PphcQuizModelBuilder' --algo-params '{ "pop_size": 1, "pareto_n": 2, "child_n": 1, "gene_size": 3}' \ +# flask quiz run -q 1 -s STEP1 --algo 'evopie.pphc_quiz_model.PphcQuizModel' --algo-params '{ "pop_size": 1, "pareto_n": 2, "child_n": 1, "gene_size": 3}' \ # --evo-output algo/pphc-1-2-1-3.json --archive-output algo/pphc-1-2-1-3-archive.csv flask deca result --algo-input algo/pphc-1-2-1-3.json --deca-space deca-spaces/space-1_1_1_1_1_1_1_1-s_20-3.json \ diff --git a/slurm/exp.sh b/slurm/exp.sh index d80c7c7..8fe29a2 100644 --- a/slurm/exp.sh +++ b/slurm/exp.sh @@ -15,4 +15,4 @@ pipenv run flask quiz deca-experiments \ --algo-folder $WORK/evopie/data-$SLURM_ARRAY_TASK_ID/algo \ --results-folder $WORK/evopie/data-$SLURM_ARRAY_TASK_ID/results \ --random-seed 23 --num-runs 30 \ - --algo '{ "id": "pphc-1-2-1-3", "algo":"evopie.pphc_quiz_model.PphcQuizModelBuilder", "pop_size": 1, "pareto_n": 2, "child_n": 1, "gene_size": 3}' \ No newline at end of file + --algo '{ "id": "pphc-1-2-1-3", "algo":"evopie.pphc_quiz_model.PphcQuizModel", "pop_size": 1, "pareto_n": 2, "child_n": 1, "gene_size": 3}' \ No newline at end of file From 9fb9322ff48e3a354d46be2b1b25ab4234121da1 Mon Sep 17 00:00:00 2001 From: dvitel Date: Sat, 11 Feb 2023 22:22:20 -0500 Subject: [PATCH 3/3] migrated rand quiz experiment slurm --- slurm/exp-rand-quiz.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100755 slurm/exp-rand-quiz.sh diff --git a/slurm/exp-rand-quiz.sh b/slurm/exp-rand-quiz.sh new file mode 100755 index 0000000..db040c5 --- /dev/null +++ b/slurm/exp-rand-quiz.sh @@ -0,0 +1,20 @@ +#!/bin/bash +#SBATCH --job-name=deca-vs-pphc +#SBATCH --time=72:00:00 +#SBATCH --output=deca-vs-pphc-%a.out +#SBATCH --mem=8G +#SBATCH --array=2-10 + +cd ~/evopie +echo "Working in $WORK/evopie/data-$SLURM_ARRAY_TASK_ID" +export EVOPIE_DATABASE_URI=sqlite:///$WORK/evopie/data-$SLURM_ARRAY_TASK_ID/db.sqlite +# export EVOPIE_DATABASE_LOG=$WORK/evopie/data-$SLURM_ARRAY_TASK_ID/db.log +export PYTHONPATH=$(pipenv run which python) +pipenv run flask quiz deca-experiments \ + --deca-spaces $WORK/evopie/data-$SLURM_ARRAY_TASK_ID/deca-spaces \ + --algo-folder $WORK/evopie/data-$SLURM_ARRAY_TASK_ID/algo \ + --results-folder $WORK/evopie/data-$SLURM_ARRAY_TASK_ID/results \ + --random-seed 23 --num-runs 30 \ + --algo '{ "id": "rand-3", "algo":"evopie.rand_quiz_model.RandomQuizModel", "n": 3}' \ + --algo '{ "id": "rand-5", "algo":"evopie.rand_quiz_model.RandomQuizModel", "n": 5}' \ + --algo '{ "id": "pphc-1-2-1-3", "algo":"evopie.pphc_quiz_model.PphcQuizModel", "pop_size": 1, "pareto_n": 2, "child_n": 1, "gene_size": 3}' \ No newline at end of file