_common.py · Scientific Agent Skills

1"""Small data and evaluation guards shared by the bundled training examples."""

2import csv

3import numpy as np

4import deepchem as dc

5from rdkit import Chem

6from sklearn.metrics import (accuracy_score, f1_score, mean_absolute_error,

7 mean_squared_error, r2_score, roc_auc_score,

8 average_precision_score)

9

10

11def load_csv(path, tasks, smiles_col, featurizer):

12 """Reject ambiguous headers, invalid SMILES/labels and silent row loss."""

13 if not tasks or len(set(tasks)) != len(tasks) or smiles_col in tasks:

14 raise ValueError('Choose distinct target columns separate from SMILES')

15 with open(path, newline='', encoding='utf-8-sig') as stream:

16 reader = csv.DictReader(stream)

17 fields = reader.fieldnames or []

18 if len(fields) != len(set(fields)):

19 raise ValueError('CSV has duplicate column headers')

20 if not set([smiles_col, *tasks]).issubset(fields):

21 raise ValueError('CSV is missing requested SMILES/target columns')

22 count = 0

23 for line, row in enumerate(reader, 2):

24 count += 1

25 if None in row or any(value is None for value in row.values()):

26 raise ValueError(f'CSV row {line} has the wrong number of fields')

27 smiles = row[smiles_col].strip()

28 molecule = Chem.MolFromSmiles(smiles)

29 if not smiles or molecule is None or molecule.GetNumAtoms() == 0:

30 raise ValueError(f'Invalid SMILES at row {line}')

31 for task in tasks:

32 value = row[task].strip()

33 if value and not np.isfinite(float(value)):

34 raise ValueError(f'Non-finite label at row {line}, task {task}')

35 if not count:

36 raise ValueError('CSV has no data rows')

37 loader = dc.data.CSVLoader(tasks=tasks, feature_field=smiles_col,

38 featurizer=featurizer)

39 dataset = loader.create_dataset(str(path))

40 if len(dataset) != count:

41 raise ValueError('Featurization dropped rows; inspect unsupported molecules')

42 return dataset

43

44

45def validate_splits(datasets, task_type, n_tasks):

46 """Check the assumptions needed for fitting and meaningful evaluation."""

47 for name, dataset in zip(('Train', 'Valid', 'Test'), datasets):

48 if not len(dataset):

49 raise ValueError(f'{name} split is empty; choose a suitable split')

50 y, w = np.asarray(dataset.y), np.asarray(dataset.w)

51 if y.shape != w.shape or y.ndim != 2 or y.shape[1] != n_tasks:

52 raise ValueError(f'{name} labels/weights have incompatible shapes')

53 if not np.isfinite(w).all() or (w < 0).any():

54 raise ValueError(f'{name} weights must be finite and nonnegative')

55 if not np.isfinite(y[w > 0]).all():

56 raise ValueError(f'{name} observed labels must be finite')

57 if name == 'Train' and (np.sum(w > 0, axis=0) == 0).any():

58 raise ValueError('Every task needs an observed training label')

59 if task_type == 'classification' and not np.isin(y[w > 0], [0, 1]).all():

60 raise ValueError('These classification examples require binary 0/1 labels')

61

62

63def evaluate_model(model, dataset, task_type, transformers=(), logits=False):

64 """Per-task metrics in original units, excluding unobserved labels.

65

66 HF 2.8.0 returns logits; graph classifiers return probabilities. Macro means

67 omit undefined tasks and report the actual number of contributing tasks.

68 Nonzero sample weights designate observed labels; this is unweighted scoring.

69 """

70 prediction = np.asarray(model.predict(dataset, transformers=list(transformers)))

71 y = dc.trans.undo_transforms(dataset.y, list(transformers))

72 if task_type == 'classification':

73 if logits:

74 shifted = prediction - prediction.max(axis=-1, keepdims=True)

75 probability = np.exp(shifted)

76 prediction = probability / probability.sum(axis=-1, keepdims=True)

77 if prediction.ndim == 2 and y.shape[1] == 1 and prediction.shape[1] == 2:

78 prediction = prediction[:, None, :]

79 if prediction.shape != (*y.shape, 2):

80 raise ValueError(f'Expected (samples, tasks, 2) class scores, got {prediction.shape}')

81 if not np.isfinite(prediction).all() or (prediction < 0).any() or (prediction > 1).any():

82 raise ValueError('Class scores must be finite probabilities')

83 prediction = prediction[..., 1]

84 else:

85 prediction = prediction.reshape(y.shape)

86 if not np.isfinite(prediction).all():

87 raise ValueError('Model returned non-finite predictions')

88 rows = []

89 for task in range(y.shape[1]):

90 observed = dataset.w[:, task] > 0

91 actual, pred = y[observed, task], prediction[observed, task]

92 scores = {'observed': int(observed.sum())}

93 if task_type == 'classification':

94 scores.update(accuracy=float(accuracy_score(actual, pred >= .5)) if len(actual) else float('nan'),

95 f1=float(f1_score(actual, pred >= .5, zero_division=0)) if len(actual) else float('nan'),

96 roc_auc=float(roc_auc_score(actual, pred)) if len(np.unique(actual)) == 2 else float('nan'),

97 average_precision=float(average_precision_score(actual, pred)) if len(np.unique(actual)) == 2 else float('nan'))

98 else:

99 scores.update(mae=float(mean_absolute_error(actual, pred)) if len(actual) else float('nan'),

100 rmse=float(np.sqrt(mean_squared_error(actual, pred))) if len(actual) else float('nan'),

101 r2=float(r2_score(actual, pred)) if len(actual) > 1 and np.ptp(actual) > 0 else float('nan'))

102 rows.append(scores)

103 result = {'per_task': rows, 'macro': {}, 'contributing_tasks': {}}

104 for key in rows[0]:

105 if key == 'observed':

106 continue

107 values = [row[key] for row in rows if np.isfinite(row[key])]

108 result['macro'][key] = float(np.mean(values)) if values else float('nan')

109 result['contributing_tasks'][key] = len(values)

110 return result

111

112

113def evaluate_splits(model, datasets, task_type, transformers=(), logits=False):

114 results = {}

115 for name, dataset in zip(('Train', 'Valid', 'Test'), datasets):

116 result = evaluate_model(model, dataset, task_type, transformers, logits)

117 results[name] = result

118 print(f'{name}: {result}')

119 return results