shadow_models

  1from typing import Dict, List, Tuple
  2
  3import utils
  4import datasets as ds
  5import target_models as tm
  6import shadow_data as sd
  7
  8from tensorflow.python.framework import random_seed
  9from tensorflow.keras import Sequential  # pyright: ignore
 10from tensorflow.data import Dataset
 11from tensorflow import keras
 12
 13import numpy as np
 14from os.path import dirname, isdir, join
 15from os import makedirs
 16
 17global_seed: int = 1234
 18
 19
 20def set_seed(new_seed: int):
 21    """
 22    Set the global seed that will be used for all functions that include
 23    randomness.
 24    """
 25    global global_seed
 26    global_seed = new_seed
 27    random_seed.set_seed(global_seed)
 28
 29
 30def get_shadow_model_name(config: Dict, i: int):
 31    numModels: int = config["shadowModels"]["number"]
 32    method: str = config["shadowDataset"]["method"] + "_"
 33    split: float = config["shadowModels"]["split"]
 34    return "shadow_" + method + tm.get_model_name(config) + f"_split_{split}_{i+1}_of_{numModels}"
 35
 36
 37def load_shadow_models_and_datasets(config: Dict) -> Tuple[List[tm.Sequential], List[Tuple[ds.Dataset, ds.Dataset]]]:
 38    verbose = config["verbose"]
 39    numModels: int = config["shadowModels"]["number"]
 40    datasets = []
 41    models = []
 42
 43    print(f"Loading shadow models from disk.")
 44    for i in range(numModels):
 45        modelName = get_shadow_model_name(config, i)
 46        model: tm.KaggleModel = load_model(modelName, verbose=verbose)
 47
 48        trainDataName = modelName + "_train_data"
 49        testDataName = modelName + "_test_data"
 50        trainData: ds.Dataset = ds.load_shadow(trainDataName, verbose=verbose)
 51        testData: ds.Dataset = ds.load_shadow(testDataName, verbose=verbose)
 52
 53        datasets.append((trainData, testData))
 54        models.append(model)
 55
 56    return models, datasets
 57
 58def load_model(name: str, verbose=True) -> Sequential:
 59    """
 60    Load model from disk.
 61
 62    The file name will be constructed from the `name` argument.
 63    """
 64    if verbose:
 65        print(f"Loading model {name} from disk.")
 66    filePath: str = join(dirname(__file__), "../models/shadow", name)
 67    return keras.models.load_model(filePath)
 68
 69def save_model(name: str, model: Sequential) -> None:
 70    """
 71    Save model to disk.
 72
 73    The file name will be constructed from the `name` argument.
 74    """
 75    folderPath: str = join(dirname(__file__),"../models/shadow")
 76    if not isdir(folderPath):
 77        makedirs(folderPath, exist_ok=True)
 78    filePath: str = join(folderPath, name)
 79    model.save(filePath)
 80
 81def train_shadow_models(config: Dict, shadowDatasets: List[ds.Dataset]
 82                        ) -> Tuple[List[tm.Sequential], List[Tuple[ds.Dataset, ds.Dataset]]]:
 83
 84    numModels: int = config["shadowModels"]["number"]
 85    split: float = config["shadowModels"]["split"]
 86    dataSize = shadowDatasets[0].cardinality().numpy()
 87    assert dataSize != 0, "Loaded shadow dataset that seems empty."
 88    trainSize = np.ceil(split * dataSize)
 89    testSize = dataSize - trainSize
 90    datasets = []
 91    models = []
 92
 93    for i in range(numModels):
 94        print(f"Training shadow model {i+1}.")
 95
 96        modelName = get_shadow_model_name(config, i)
 97        trainDataName = modelName + "_train_data"
 98        testDataName = modelName + "_test_data"
 99
100        dataset = shadowDatasets[i]
101        trainData = dataset.take(trainSize)
102        testData = dataset.skip(trainSize).take(testSize)
103
104        # Shadow models have same architecture as target model
105        model = tm.KaggleModel(config["targetModel"]["classes"])
106        modelConfig = config["targetModel"]["hyperparameters"]
107
108        tm.train_model(model, modelName, trainData, testData, modelConfig)
109
110        print(f"Saving shadow model {i+1} and its data to disk.")
111        save_model(modelName, model)
112        ds.save_shadow(trainData, trainDataName)
113        ds.save_shadow(testData, testDataName)
114
115        print(f"Evaluating shadow model {i+1}")
116        tm.evaluate_model(model, testData)
117
118        datasets.append((trainData, testData))
119        models.append(model)
120
121    return models, datasets
122
123
124def get_shadow_models_and_datasets(config: Dict, shadowDatasets: List[ds.Dataset]
125                                   ) -> Tuple[List[tm.Sequential], List[Tuple[ds.Dataset, ds.Dataset]]]:
126    """
127    Tries to load shadow datasets from disk, alternatively trains from scratch.
128
129    Returns 2 lists:
130        models: the trained shadow models and a list of tuples, containing
131        datasets: the training and test data for the corresponding shadow models
132
133        E.g. models[0] is trained with datasets[0,0] and tested on datasets[0,1]
134    """
135    try:
136        print("Trying to load shadow models and data from disk.")
137        models, datasets = load_shadow_models_and_datasets(config)
138    except BaseException:
139        print("Didn't work, training shadow models.")
140        models, datasets = train_shadow_models(config, shadowDatasets)
141
142    return models, datasets
143
144def evaluate_model(model: Sequential, dataset: Dataset):
145    # TODO: batchSize is hardcoded
146    batchSize = 10
147    dataset = dataset.batch(batchSize, drop_remainder=False)
148    return model.evaluate(dataset)
149
150def evaluate_models(models:List[tm.Sequential], datasets: List[Tuple[ds.Dataset, ds.Dataset]]):
151    assert len(models) == len(datasets)
152    test_accuracies = []
153    train_accuracies = []
154    for i in range(len(models)):
155        testData = datasets[i][1]
156        trainData = datasets[i][0]
157        test_accuracy = evaluate_model(models[i], testData)[1]
158        train_accuracy = evaluate_model(models[i], trainData)[1]
159        test_accuracies.append(test_accuracy)
160        train_accuracies.append(train_accuracy)
161
162    hash = utils.hash(str(config))
163
164    with open(f"{hash}_shadowModelTrainAccuracy.csv",'w') as file:
165        file.write(f"Shadow Model Training Accuracies (Overall:{np.average(train_accuracies)})\n")
166        for train_acc in train_accuracies:
167            file.write(f"{train_acc}\n")
168    with open(f"{hash}_shadowModelTestAccuracy.csv",'w') as file:
169        file.write(f"Shadow Model Testing Accuracies (Overall:{np.average(test_accuracies)})\n")
170        for test_acc in test_accuracies:
171            file.write(f"{test_acc}\n")
172
173if __name__ == "__main__":
174    import argparse
175    import configuration as con
176    import datasets as ds
177    import target_models as tm
178    import shadow_data as sd
179
180    parser = argparse.ArgumentParser(description='Save one shadow dataset per model and train the models.')
181    parser.add_argument('--config', help='Relative path to config file.',)
182    config = con.from_cli_options(vars(parser.parse_args()))
183    set_seed(config["seed"])
184
185    shadowData = sd.load_shadow_data(config)
186    shadowDatasets = sd.split_shadow_data(config, shadowData)
187    shadowModels, shadowDatasets = get_shadow_models_and_datasets(config, shadowDatasets)
188    evaluate_models(shadowModels,shadowDatasets)