sampling quiz model

.vscode/launch.json

@@ -160,6 +160,27 @@
                 "--random-seed", "17"
             ]
         },
+        {
+            "name": "CLI: quiz run deca-pphc Sampling",
+            "type": "python",
+            "request": "launch",
+            "module" : "flask",
+            "env" : {
+                "FLASK_APP" : "app.py",
+                "FLASK_ENV" : "development",
+                "FLASK_DEBUG" : "1",
+                // "EVOPIE_DATABASE_LOG": "db.log" 
+            },
+            "args" : [
+                "quiz",
+                "run",
+                "-q", "1",
+                "-s", "STEP1",
+                "--algo", "evopie.sampling_quiz_model.SamplingQuizModel", "--algo-params", "{ \"n\": 3, \"min_num_evals\": 1, \"group_size\": 2, \"strategy\": \"non_domination\"}",
+                "--evo-output", "algo/s-nond.json", "--archive-output", "algo/a.csv",
+                "--random-seed", "17"
+            ]
+        },
         {
             "name": "CLI: deca init",
             "type": "python",
@@ -238,7 +259,7 @@
                 "--result-folder", "data/data-8/results",
                 "--figure-folder", "figures",
                 "--file-name-pattern", ".*_20-\\d+.csv",
-                "-p", "dim_coverage_with_spanned", "--group-by-space"
+                "--group-by-space"
             ]
         },
         {
@@ -280,6 +301,24 @@
             ],
             "console": "internalConsole"
         },
+        {
+            "name": "CLI: quiz deca-experiment Sampling",
+            "type": "python",
+            "request": "launch",
+            "module": "flask",
+            "env" : {
+                "FLASK_APP" : "app.py",
+                "FLASK_ENV" : "development",
+                "FLASK_DEBUG" : "1"
+            },
+            "args" : [
+                "quiz", "deca-experiment",
+                "--deca-input", "deca-spaces/space-4_4_4_4-s_4-0.json",
+                "--num-runs", "2",
+                "--algo", "{ \"id\": \"s-nond\", \"algo\":\"evopie.sampling_quiz_model.SamplingQuizModel\", \"n\": 3, \"min_num_evals\": 1, \"group_size\": 2, \"strategy\": \"non_domination\"}"
+            ],
+            "console": "internalConsole"
+        },
         {
             "name": "CLI: quiz deca-experiment RAND",
             "type": "python",

evopie/cli.py

@@ -455,12 +455,13 @@ def export_student_knowledge(output):
 
 KNOWLEDGE_SELECTION_CHANCE = "KNOWLEDGE_SELECTION_CHANCE"
 KNOWLEDGE_SELECTION_WEIGHT = "KNOWLEDGE_SELECTION_WEIGHT"
+KNOWLEDGE_SELECTION_STRENGTH = "KNOWLEDGE_SELECTION_STRENGTH"
 
 @quiz_cli.command("run")
 @click.option('-q', '--quiz', type=int, required=True)
 @click.option('-s', '--step', default = [QUIZ_STEP1], multiple=True)
 @click.option('-n', '--n-times', type=int, default=1)
-@click.option('-kns', '--knowledge-selection', default = KNOWLEDGE_SELECTION_WEIGHT)
+@click.option('-kns', '--knowledge-selection', default = KNOWLEDGE_SELECTION_STRENGTH)
 @click.option('-i', '--instructor', default='i@usf.edu')
 @click.option('-p', '--password', default='pwd') 
 @click.option('--no-algo', is_flag=True) #randomize student order
@@ -537,6 +538,12 @@ def simulate_step(step):
                                             for sums in [np.cumsum(weights)]
                                             for level in [(rnd_state.rand() * sums[-1]) if len(sums) > 0 else None]
                                             for selected_d_index in [next((i for i, s in enumerate(sums) if s > level), None)]}
+                elif knowledge_selection == KNOWLEDGE_SELECTION_STRENGTH:
+                    responses = {qid:sorted(ds, key=lambda x: x[1])[-1][0]
+                                            for qid, distractors in attempt.alternatives_map.items() 
+                                            for distractor_strength in [student_knowledge.get(qid, {}) ]
+                                            for ds_distr in [[(alt, distractor_strength[d]) for alt, d in enumerate(distractors) if d in distractor_strength]] 
+                                            for ds in [ds_distr if any(ds_distr) else [(alt,1) for alt, d in enumerate(distractors) if d == -1]]}
                 else: 
                     responses = {}
 
@@ -573,17 +580,19 @@ def simulate_step(step):
                 if quiz_model is None:
                     sys.stdout.write(f"[{run_idx + 1}/{n_times}] Step1 quiz {quiz} finished\n")            
                 else:
-                    model_state = quiz_model.get_model_state()
+                    settings = quiz_model.get_model_state()
+                    best_distractors = quiz_model.get_best_quiz()
 
                     if archive_output is not None: 
                         quiz_model.to_csv(archive_output.format(run_idx))
 
                     search_space_size = quiz_model.get_search_space_size()
                     explored_search_space_size = quiz_model.get_explored_search_space_size()
-                    sys.stdout.write(f"EVO algo: {quiz_model.__class__}\nEVO settings: {model_state}\n")
+                    sys.stdout.write(f"EVO algo: {quiz_model.__class__}\nEVO settings: {settings}\n")
                     if evo_output: 
                         with open(evo_output, 'w') as evo_output_json_file:
-                            evo_output_json_file.write(json.dumps({**model_state, "algo": quiz_model.__class__.__name__, 
+                            evo_output_json_file.write(json.dumps({"settings": settings, "distractors": best_distractors, 
+                                                        "algo": quiz_model.__class__.__name__, 
                                                         "explored_search_space_size": explored_search_space_size,
                                                         "search_space_size": search_space_size}, indent=4))
                     sys.stdout.write(f"[{run_idx + 1}/{n_times}] Step1 quiz {quiz} finished\n{quiz_model.to_dataframe()}\n")

evopie/deca.py

@@ -260,12 +260,12 @@ def unique_question_per_axis_constraint(space):
 def gen_test_distractor_mapping_knowledge(space): 
     ''' Generates knowledge in format similar to cli student init -k flag. Use this to init StudentKnowledge database 
     '''
-    knowledge = [*[ {"sid": point['cfs'], "qid": qid, "did": did, "chance": 1 } 
+    knowledge = [*[ {"sid": point['cfs'], "qid": qid, "did": did, "chance": 100 + int(point_id) } 
                     for _, points in space['axes'].items()
-                    for _, point in points.items() 
+                    for point_id, point in points.items() 
                     for qid, did in point['dids']],
-                 *[ {"sid": point['cfs'], "qid": qid, "did": did, "chance": 1 } 
-                    for _, point in space['spanned'].items()
+                 *[ {"sid": point['cfs'], "qid": qid, "did": did, "chance": 100 * len(axes_ids) + len(point['dids']) } 
+                    for axes_ids, point in space['spanned'].items()
                     for qid, did in point['dids']]]
     return knowledge
 
@@ -282,8 +282,8 @@ def dimension_coverage(space, population_distractors):
     spanned_axes = {axis_id for spanned_id, point in space['spanned'].items() for _, did in point['dids'] if did in population_distractors for axis_id, _ in spanned_id}
     did_axes = {axis_id for axis_id, points in space["axes"].items() for point in points.values() for _, did in point["dids"] if did in population_distractors}    
     dim_coverage = len(did_axes) / len(space["axes"])
-    dim_coverage_with_spanned = len(did_axes.union(spanned_axes)) / len(space["axes"])
-    return {"dim_coverage":dim_coverage, "dim_coverage_with_spanned": dim_coverage_with_spanned}
+    # dim_coverage_with_spanned = len(did_axes.union(spanned_axes)) / len(space["axes"])
+    return {"dim_coverage":dim_coverage} # "dim_coverage_with_spanned": dim_coverage_with_spanned}
 
 # dimension_coverage(space, [5,6,14])
 
@@ -301,10 +301,10 @@ def avg_rank_of_repr(space, population_distractors):
                                                                         if did in population_distractors], key = lambda x: x[0])}
     arr_axes = [axes_rank[axis_id] / len(space['axes'][axis_id]) for axis_id in space['axes'] if axis_id in axes_rank]
     arr = np.average(arr_axes) if len(arr_axes) > 0 else 0 
-    arr_with_spanned_axes = [max([axes_rank.get(axis_id, 0), spanned_ranks.get(axis_id, 0)]) / len(space['axes'][axis_id]) 
-                                for axis_id in space['axes'] if axis_id in axes_rank or axis_id in spanned_ranks]
-    arr_with_spanned = np.average(arr_with_spanned_axes) if len(arr_with_spanned_axes) > 0 else 0 
-    return {"arr": arr, "arr_with_spanned": arr_with_spanned}
+    # arr_with_spanned_axes = [max([axes_rank.get(axis_id, 0), spanned_ranks.get(axis_id, 0)]) / len(space['axes'][axis_id]) 
+    #                             for axis_id in space['axes'] if axis_id in axes_rank or axis_id in spanned_ranks]
+    # arr_with_spanned = np.average(arr_with_spanned_axes) if len(arr_with_spanned_axes) > 0 else 0 
+    return {"arr": arr}#, "arr_with_spanned": arr_with_spanned}
 
 # avg_rank_of_repr(space, [5,6,14])
 
@@ -314,7 +314,7 @@ def redundancy(space, population_distractors):
     population_spanned_distractors = [did for _, point in space['spanned'].items() for _, did in point['dids'] if did in population_distractors]
     pop_r = len(population_spanned_distractors) / len(population_distractors) 
     deca_r = len(space['spanned']) / (len(space['spanned']) + sum([len(points) for points in space["axes"].values()]))
-    return {"population_redundancy":pop_r, "deca_redundancy":deca_r, 'num_spanned': len(space['spanned']) }
+    return {"population_redundancy":pop_r, "deca_redundancy":deca_r} #, 'num_spanned': len(space['spanned']) }
 
 # redundancy(space, [5,6,14])
 

evopie/pphc_quiz_model.py

@@ -260,8 +260,6 @@ def prepare_for_evaluation(self, evaluator_id: int) -> 'list[tuple[int, list[int
         return question_distractors
 
     def get_model_state(self):
-        population = [self.archive.get(genotype_id) for genotype_id in self.population]
-        distractors = [d for genotype in population for (_, dids) in genotype for d in dids ] 
         archive = [ {"genotype_id": genotype_id, "genotype": r.g.g, "objectives": {c: r[c] for c in r[r.notnull()].index if c != "g" }}
                                             for genotype_id, r in self.archive.items() ]
         settings = { "pop_size": self.pop_size, "gen": self.gen, "seed": self.seed,
@@ -271,7 +269,12 @@ def get_model_state(self):
                     "population": self.population, 
                     "objectives": self.objectives,
                     "archive": archive}   
-        return {"population": population, "distractors": distractors, "settings": settings}
+        return settings
+
+    def get_best_quiz(self):
+        population = [self.archive.get(genotype_id) for genotype_id in self.population]
+        distractors = [d for genotype in population for (_, dids) in genotype for d in dids ] 
+        return distractors
 
     def update_fitness(self, ind: int, evaluator_id: int, result: 'dict[int, int]') -> None:  
         # cur_score = self.archive.loc[ind, evaluator_id]

evopie/quiz_model.py

@@ -52,19 +52,24 @@ def evaluate(self, evaluator_id: int, result: 'dict[int, int]') -> None:
         self.save()
     def save(self) -> None:
         ''' preserve state in persistent storage'''
-        impl_state = self.get_model_state().get("settings", {})
+        impl_state = self.get_model_state()
         self.process.impl = self.__class__ .__module__ + '.' + self.__class__.__name__
         self.process.impl_state = impl_state
         models.DB.session.commit()         
-    def to_csv(self, file_name) -> None: 
-        ''' save state to csv file '''
-        pass 
     def to_dataframe(self) -> Optional[DataFrame]: 
         return None
+    def to_csv(self, file_name) -> None: 
+        ''' save state to csv file '''
+        df = self.to_dataframe()
+        if df is not None:
+            df.to_csv(file_name)
+    def get_best_quiz(self):
+        ''' Should return known best quiz distractors '''
+        return []
 
-from evopie.pphc_quiz_model import PphcQuizModel
+from evopie.sampling_quiz_model import SamplingQuizModel
 class QuizModelBuilder():
-    default_quiz_model_class = PphcQuizModel
+    default_quiz_model_class = SamplingQuizModel
     default_settings = {}
 
     def get_quiz(self, quiz_or_id: 'models.Quiz | int') -> models.Quiz: 

evopie/rand_quiz_model.py

@@ -2,6 +2,7 @@
 from evopie import models
 import numpy as np
 from evopie.quiz_model import QuizModel
+from pandas import DataFrame
 
 class RandomQuizModel(QuizModel):
     ''' Implements dummy choice of quiz questions - random questions at start '''
@@ -26,12 +27,18 @@ def get_sampling_size(self):
         return self.n
 
     def get_model_state(self) -> Any: #depends on impl, returns population for Evo models
-        population = [self.quiz_model]
-        distractors = [d for genotype in population for (_, dids) in genotype for d in dids ] 
-        return {"settings": {'n':self.n, 'seed': self.seed, 'quiz_model': self.quiz_model}, "population": population, "distractors": distractors}
+        return {'n':self.n, 'seed': self.seed, 'quiz_model': self.quiz_model}
 
     def prepare_for_evaluation(self, evaluator_id: int) -> 'list[tuple[int, list[int]]]':
         return [(q, list(ds)) for [q, ds] in self.quiz_model]
 
     def evaluate_internal(self, evaluator_id: int, result: 'dict[int, int]'):
-        self.interactions[str(evaluator_id)] = sum([1 if result.get(qid, -1) in genes else 0 for qid, genes in self.quiz_model ])
+        self.interactions[str(evaluator_id)] = sum([1 if result.get(qid, -1) in genes else 0 for qid, genes in self.quiz_model ])
+
+    def to_dataframe(self):
+        return DataFrame(data = [{'g': self.quiz_model, **self.interactions}])
+
+    def get_best_quiz(self):
+        population = [self.quiz_model]
+        distractors = [d for genotype in population for (_, dids) in genotype for d in dids ] 
+        return distractors