attack_model

  1"""
  2.. include:: ../docs/attack_model.md
  3"""
  4
  5from os import environ
  6
  7# Tensorflow C++ backend logging verbosity
  8environ["TF_CPP_MIN_LOG_LEVEL"] = "2"  # NOQA
  9
 10from os.path import dirname, join
 11
 12from typing import Dict, List, Tuple
 13from tensorflow import keras
 14from tensorflow.data import Dataset  # pyright: ignore
 15from tensorflow.python.framework import random_seed
 16from tensorflow.keras.activations import relu  # pyright: ignore
 17from tensorflow.keras.initializers import glorot_uniform  # pyright: ignore
 18from tensorflow.keras.layers import Dense, InputLayer, Softmax  # pyright: ignore
 19from tensorflow.keras import Sequential  # pyright: ignore
 20import numpy as np
 21
 22import utils
 23import datasets as ds
 24
 25global_seed: int = 1234
 26
 27
 28def set_seed(new_seed: int):
 29    """
 30    Set the global seed that will be used for all functions that include
 31    randomness.
 32    """
 33    global global_seed
 34    global_seed = new_seed
 35    random_seed.set_seed(global_seed)
 36
 37
 38class KaggleAttackModel(Sequential):
 39    """
 40    Architecture:
 41        Fully connected NN,
 42        1 hiddenlayer, size 64,
 43        ReLU activation
 44        Softmax LAyer
 45
 46    One model for each class
 47    """
 48
 49    def __init__(self, numClasses: int) -> None:
 50        super().__init__()
 51        activation = relu
 52        initializer = glorot_uniform
 53        self.add(InputLayer(input_shape=(numClasses)))
 54        self.add(Dense(64, activation=activation,
 55                 kernel_initializer=initializer))
 56        self.add(Dense(2, kernel_initializer=initializer))
 57        self.add(Softmax())
 58
 59
 60def load_model(name: str, verbose=True) -> Sequential:
 61    """
 62    Load model from disk.
 63
 64    The file name will be constructed from the `name` argument.
 65    """
 66    if verbose:
 67        print(f"Loading model {name} from disk.")
 68    filePath: str = join(dirname(__file__), "../models/attack", name)
 69    return keras.models.load_model(filePath)
 70
 71
 72def save_model(name: str, model: Sequential) -> None:
 73    """
 74    Save model to disk.
 75
 76    The file name will be constructed from the `name` argument.
 77    """
 78    filePath: str = join(dirname(__file__), "../models/attack", name)
 79    model.save(filePath)
 80
 81
 82def train_model(model: Sequential, modelName: str, trainData: Dataset,
 83                testData: Dataset, hyperpar: Dict):
 84    epochs: int = int(hyperpar["epochs"])
 85    learningRate: float = float(hyperpar["learningRate"])
 86    batchSize: int = int(hyperpar["batchSize"])
 87
 88    optimizer = keras.optimizers.Adam(name="Adam", learning_rate=learningRate)
 89    loss = keras.losses.CategoricalCrossentropy()
 90    metrics = ["accuracy"]
 91
 92    model.compile(optimizer, loss, metrics)
 93    # TODO: drop_remainder: make sure dataset is still 50/50 in/out
 94    trainData = trainData.batch(batchSize, drop_remainder=True)
 95    testData = testData.batch(batchSize, drop_remainder=True)
 96    log_dir = "logs/attack/" + modelName
 97    cb = keras.callbacks.TensorBoard(histogram_freq=1, log_dir=log_dir)
 98    return model.fit(trainData, epochs=epochs, callbacks=[cb], validation_data=testData)
 99
100
101def evaluate_models(models: List[Sequential], datasets: List[ds.Dataset]):
102    # TODO: Evaluate on randomly reshuffled records from test/train dataset
103    assert len(models) == len(datasets)
104    test_accuracies = []
105    train_accuracies = []
106    for i in range(len(models)):
107        testData = datasets[i][0]
108        trainData = datasets[i][1]
109        test_accuracy = evaluate_model(models[i], testData)[1]
110        train_accuracy = evaluate_model(models[i], trainData)[1]
111        test_accuracies.append(test_accuracy)
112        train_accuracies.append(train_accuracy)
113
114    hash = utils.hash(str(config))
115
116    with open(f"{hash}_attackModelTrainAccuracy.csv",'w') as file:
117        file.write(f"Attack Model Training Accuracies (Overall:{np.average(train_accuracies)})\n")
118        for train_acc in train_accuracies:
119            file.write(f"{train_acc}\n")
120    with open(f"{hash}_attackModelTestAccuracy.csv",'w') as file:
121        file.write(f"Attack Model Testing Accuracies (Overall:{np.average(test_accuracies)})\n")
122        for test_acc in test_accuracies:
123            file.write(f"{test_acc}\n")
124
125
126def evaluate_model(model: Sequential, dataset: Dataset):
127    # TODO: batchSize is hardcoded
128    batchSize = 10
129    dataset = dataset.batch(batchSize, drop_remainder=False)
130    return model.evaluate(dataset)
131
132
133def get_model_name(config: Dict, i: int) -> str:
134    modelConfig = config["attackModel"]["hyperparameters"]
135    numClasses = config["targetModel"]["classes"]
136    return \
137        f'{config["targetDataset"]["name"]}_' + \
138        f'{config["shadowDataset"]["method"]}_' + \
139        f'lr_{modelConfig["learningRate"]}_' + \
140        f'bs_{modelConfig["batchSize"]}_' + \
141        f'epochs_{modelConfig["epochs"]}_' + \
142        f'{i+1}_of_{numClasses}'
143
144
145def get_attack_models(config: Dict, attackDatasets: List[Tuple[ds.Dataset, ds.Dataset]]) -> List[KaggleAttackModel]:
146    verbose = config["verbose"]
147    modelConfig = config["attackModel"]["hyperparameters"]
148    numClasses = config["targetModel"]["classes"]
149    attackModels = []
150
151    print(f"Loading attack models from disk.")
152    for i in range(numClasses):
153        modelName = get_model_name(config, i)
154        try:
155            model: KaggleAttackModel = load_model(modelName, verbose=verbose)
156        except BaseException:
157            print(f"Couldn't load attack model {i+1}, retraining.")
158            testData, trainData = attackDatasets[i]
159            trainData = ds.shuffle(trainData)
160
161            model = KaggleAttackModel(config["targetModel"]["classes"])
162
163            train_model(model, modelName, trainData, testData, modelConfig)
164
165            print(f"Saving attack model {i+1} to disk.")
166            model._name = modelName
167            save_model(modelName, model)
168            evaluate_model(model, testData)
169
170        attackModels.append(model)
171
172    return attackModels
173
174
175if __name__ == "__main__":
176    import argparse
177    import configuration as con
178    import attack_data as ad
179
180    parser = argparse.ArgumentParser(description='Train the attack models.')
181    parser.add_argument('--config', help='Relative path to config file.',)
182    config = con.from_cli_options(vars(parser.parse_args()))
183    set_seed(config["seed"])
184
185    attackDatasets = ad.load_attack_data(config)
186    attackModels = get_attack_models(config, attackDatasets)
187    evaluate_models(attackModels, attackDatasets)