王半仙的学习分享

import optuna
def objective(trial):
    x = trial.suggest_float("x", -10, 10) # 定义待优化参数及其值域
    return (x - 2) ** 2 # 定义目标函数

study = optuna.create_study()
study.optimize(objective, n_trials=100)
best_params = study.best_params
found_x = best_params["x"]
print("Found x: {}, (x - 2)^2: {}".format(found_x, (found_x - 2) ** 2))
# 结果：Found x: 1.9918889682901693, (x - 2)^2: 6.578883539787978e-05

import optuna

# 搜索空间：包含不同的参数类型
def objective(trial):
    # Categorical parameter 
    optimizer = trial.suggest_categorical("optimizer", ["MomentumSGD", "Adam"])
    # Integer parameter
    num_layers = trial.suggest_int("num_layers", 1, 3)
    # Integer parameter (log)
    num_channels = trial.suggest_int("num_channels", 32, 512, log=True)
    # Integer parameter (discretized)
    num_units = trial.suggest_int("num_units", 10, 100, step=5)
    # Floating point parameter
    dropout_rate = trial.suggest_float("dropout_rate", 0.0, 1.0)
    # Floating point parameter (log)
    learning_rate = trial.suggest_float("learning_rate", 1e-5, 1e-2, log=True)
    # Floating point parameter (discretized)
    drop_path_rate = trial.suggest_float("drop_path_rate", 0.0, 1.0, step=0.1)

import sklearn.ensemble
import sklearn.svm

# 搜索空间：通过条件语句区分不同模型的不同超参
def objective(trial):
    classifier_name = trial.suggest_categorical("classifier", ["SVC", "RandomForest"])
    if classifier_name == "SVC":
        svc_c = trial.suggest_float("svc_c", 1e-10, 1e10, log=True)
        classifier_obj = sklearn.svm.SVC(C=svc_c)
    else:
        rf_max_depth = trial.suggest_int("rf_max_depth", 2, 32, log=True)
        classifier_obj = sklearn.ensemble.RandomForestClassifier(max_depth=rf_max_depth)

import torch
import torch.nn as nn

# 搜索空间：通过循环语句区分不同神经网络层的不同超参
def create_model(trial, in_size):
    n_layers = trial.suggest_int("n_layers", 1, 3)

    layers = []
    for i in range(n_layers):
        n_units = trial.suggest_int("n_units_l{}".format(i), 4, 128, log=True)
        layers.append(nn.Linear(in_size, n_units))
        layers.append(nn.ReLU())
        in_size = n_units
    layers.append(nn.Linear(in_size, 10))
    return nn.Sequential(*layers)

import logging
import sys

import sklearn.datasets
import sklearn.linear_model
import sklearn.model_selection


def objective(trial):
    iris = sklearn.datasets.load_iris()
    classes = list(set(iris.target))
    train_x, valid_x, train_y, valid_y = sklearn.model_selection.train_test_split(
        iris.data, iris.target, test_size=0.25, random_state=0
    )

    alpha = trial.suggest_float("alpha", 1e-5, 1e-1, log=True)
    clf = sklearn.linear_model.SGDClassifier(alpha=alpha)

    for step in range(100):
        clf.partial_fit(train_x, train_y, classes=classes)

        # Report intermediate objective value.
        intermediate_value = 1.0 - clf.score(valid_x, valid_y)
        trial.report(intermediate_value, step)

        # Handle pruning based on the intermediate value.
        if trial.should_prune(): # 使用剪枝算法
            raise optuna.TrialPruned()

    return 1.0 - clf.score(valid_x, valid_y)

# Add stream handler of stdout to show the messages
optuna.logging.get_logger("optuna").addHandler(logging.StreamHandler(sys.stdout))
study = optuna.create_study(pruner=optuna.pruners.MedianPruner(), # 指定采样+剪枝方法
    					   sampler=optuna.samplers.CmaEsSampler())
study.optimize(objective, n_trials=20)

mysql -u root -e "CREATE DATABASE IF NOT EXISTS example"
optuna create-study --study-name "distributed-example" --storage "mysql://root@localhost/example"

import optuna

def objective(trial):
    x = trial.suggest_float("x", -10, 10)
    return (x - 2) ** 2

if __name__ == "__main__":
    study = optuna.load_study(
        study_name="distributed-example", storage="mysql://root@localhost/example"
    )
    study.optimize(objective, n_trials=100)

pip install optuna-dashboard
optuna-dashboard sqlite:///example-study.db

import lightgbm as lgb
import numpy as np
import sklearn.datasets
import sklearn.metrics
from sklearn.model_selection import train_test_split
import optuna

# You can use Matplotlib instead of Plotly for visualization by simply replacing `optuna.visualization` with
# `optuna.visualization.matplotlib` in the following examples.
from optuna.visualization import plot_contour
from optuna.visualization import plot_edf
from optuna.visualization import plot_intermediate_values
from optuna.visualization import plot_optimization_history
from optuna.visualization import plot_parallel_coordinate
from optuna.visualization import plot_param_importances
from optuna.visualization import plot_slice

SEED = 42
np.random.seed(SEED)

def objective(trial):
    data, target = sklearn.datasets.load_breast_cancer(return_X_y=True)
    train_x, valid_x, train_y, valid_y = train_test_split(data, target, test_size=0.25)
    dtrain = lgb.Dataset(train_x, label=train_y)
    dvalid = lgb.Dataset(valid_x, label=valid_y)

    param = {
        "objective": "binary",
        "metric": "auc",
        "verbosity": -1,
        "boosting_type": "gbdt",
        "bagging_fraction": trial.suggest_float("bagging_fraction", 0.4, 1.0),
        "bagging_freq": trial.suggest_int("bagging_freq", 1, 7),
        "min_child_samples": trial.suggest_int("min_child_samples", 5, 100),
    }

    # Add a callback for pruning.
    pruning_callback = optuna.integration.LightGBMPruningCallback(trial, "auc")
    gbm = lgb.train(param, dtrain, valid_sets=[dvalid], callbacks=[pruning_callback])

    preds = gbm.predict(valid_x)
    pred_labels = np.rint(preds)
    accuracy = sklearn.metrics.accuracy_score(valid_y, pred_labels)
    return accuracy

study = optuna.create_study(
    direction="maximize",
    sampler=optuna.samplers.TPESampler(seed=SEED),
    pruner=optuna.pruners.MedianPruner(n_warmup_steps=10),
)
study.optimize(objective, n_trials=100, timeout=600)

plot_optimization_history(study) # 可视化trial的优化历史（每一次trial的最优结果）
plot_intermediate_values(study) # 可视化每一次trial的所有迭代结果（学习曲线）
plot_parallel_coordinate(study) # 可视化高维参数
plot_contour(study) # 可视化参数间关系
plot_slice(study) # 绘制每一个参数的切片图（不同参数值对应的目标值）
plot_param_importances(study) # 绘制不同参数的重要性
optuna.visualization.plot_param_importances(
    study, target=lambda t: t.duration.total_seconds(), target_name="duration"
) # 绘制不同参数的模拟耗时
plot_edf(study) # 绘制目标函数的经验分布函数图像