# "utiml" and "mldr" packages for multi-label classification in R
# https://journal.r-project.org/archive/2018/RJ-2018-041/RJ-2018-041.pdf
# https://github.com/rivolli/utiml
# https://cran.r-project.org/web/packages/mldr/vignettes/mldr.pdf
# https://github.com/fcharte/mldr
library(utiml)
Loading required package: mldr
Enter mldrGUI() to launch mldr's web-based GUI
Loading required package: parallel
Loading required package: ROCR
library(mldr)
# following package offers benchmarks for multi-label classification
library(mldr.datasets)

Attaching package: ‘mldr.datasets’

The following object is masked _by_ ‘.GlobalEnv’:

    ng20

The following object is masked from ‘package:stats’:

    density
set.seed(123)

1 ng20 Dataset

summary(ng20)
# ng20, a corpus with 19300 documents, 1006 words and 20 multi-labels
# Ken Lang, "Newsweeder: Learning to filter netnews", 12th ICML Conference
ng20$labels
# not practical, but part of its corpus can be viewed doing
ng20corpus <- ng20$dataset
# ng20's bag of words
dim(ng20corpus)
[1] 19300  1028
# colnames(ng20corpus)
# consult the help of the following function
ng20 <- remove_skewness_labels(ng20, 10)
# label bat plot
plot(ng20, type = "LB")

# visual relations among labels
plot(ng20, type = "LC")
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.

# create a holdout partition: train and predict to evaluate
# I created a small test partition as the prediction step takes a long time
ds <- create_holdout_partition(ng20, c(train = 0.90, test = 0.10))
# an external GUI interface to explore the "ng20" dataset
# mldrGUI() # press "escape" to exit GUI

NG20 labels

NG20 labelsets

NG20 concurrence

1.1 Binary Relevance Naive Bayes

# Binary relevance ML strategy with naive Bayes base classifier
model_BR_NB <- utiml::br(ds$train, "NB")
predictionsBR <- predict(model_BR_NB, ds$test)
resultsBRPerExamples <- multilabel_evaluate(
    ds$test, predictionsBR,
    c("example-based")
)
resultsBRPerExamples
       accuracy              F1    hamming-loss       precision          recall 
     0.13449404      0.21632577      0.52443005      0.13462829      0.97694301 
subset-accuracy 
     0.01398964 
resultsBRPerLabel <- multilabel_evaluate(
    ds$test, predictionsBR,
    c("label-based")
)
resultsBRPerLabel
      macro-AUC        macro-F1 macro-precision    macro-recall       micro-AUC 
     0.72405297      0.16150400      0.08811967      0.97653916      0.72425348 
       micro-F1 micro-precision    micro-recall 
     0.16091192      0.08767730      0.97684952 

1.2 Classifier Chain Naive Bayes

# Classifier Chain ML strategy with naive Bayes base classifier
# Define the chain-order between labels: sample a random order
mychain <- sample(rownames(ng20$labels))
model_CC_NB <- cc(ds$train, "NB", mychain)
predictionsCC <- predict(model_CC_NB, ds$test)
resultsCCPerExamples <- multilabel_evaluate(
    ds$test, predictionsCC,
    c("example-based")
)
resultsCCPerExamples
       accuracy              F1    hamming-loss       precision          recall 
     0.13706745      0.22035917      0.50878238      0.13720169      0.97590674 
subset-accuracy 
     0.01398964 
resultsCCPerLabel <- multilabel_evaluate(
    ds$test, predictionsCC,
    c("label-based")
)
resultsCCPerLabel
      macro-AUC        macro-F1 macro-precision    macro-recall       micro-AUC 
     0.73162938      0.16577219      0.09068529      0.97540831      0.73184878 
       micro-F1 micro-precision    micro-recall 
     0.16490199      0.09006038      0.97584298 

2 birds Dataset

# "birds" dataset to predict the set of birds species
# that are present, given a ten-second audio clip:
# https://doi.org/10.1109/MLSP.2013.6661934
summary(birds)
birds$labels
birds <- remove_skewness_labels(birds, 10)
# Type "birds$" for its list of attributes
plot(birds, type = "LB")

# visual relations among labels
plot(birds, type = "LC")
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.

# create a holdout partition: train and predict to evaluate
ds <- create_holdout_partition(birds, c(train = 0.66, test = 0.34))
# an external GUI interface to explore the "birds" dataset
# mldrGUI() # press "escape" to exit GUI

Birds labels

Birds labelsets

Birds concurrence

2.1 Binary Relevance Naive Bayes

# Binary relevance ML strategy with naive Bayes base classifier
model_BR_NB <- utiml::br(ds$train, "NB")
predictionsBR <- predict(model_BR_NB, ds$test)
resultsBRPerExamples <- multilabel_evaluate(
    ds$test, predictionsBR,
    c("example-based")
)
resultsBRPerExamples
       accuracy              F1    hamming-loss       precision          recall 
     0.07904289      0.12978222      0.69041096      0.08127791      0.44406393 
subset-accuracy 
     0.00000000 
resultsBRPerLabel <- multilabel_evaluate(
    ds$test, predictionsBR,
    c("label-based")
)
resultsBRPerLabel
      macro-AUC        macro-F1 macro-precision    macro-recall       micro-AUC 
     0.57931277      0.13520211      0.07721916      0.85064474      0.56851048 
       micro-F1 micro-precision    micro-recall 
     0.13236419      0.07172471      0.85643564 

2.2 Classifier Chain Naive Bayes

# Classifier Chain ML strategy with naive Bayes base classifier
# Define the chain-order between labels: sample a random order
mychain <- sample(rownames(birds$labels))
model_CC_NB <- cc(ds$train, "NB", mychain)
predictionsCC <- predict(model_CC_NB, ds$test)
resultsCCPerExamples <- multilabel_evaluate(
    ds$test, predictionsCC,
    c("example-based")
)
resultsCCPerExamples
       accuracy              F1    hamming-loss       precision          recall 
     0.07944093      0.13031361      0.69406393      0.08168517      0.44406393 
subset-accuracy 
     0.00000000 
resultsCCPerLabel <- multilabel_evaluate(
    ds$test, predictionsCC,
    c("label-based")
)
resultsCCPerLabel
      macro-AUC        macro-F1 macro-precision    macro-recall       micro-AUC 
     0.58031688      0.13302412      0.07543980      0.85064474      0.57556851 
       micro-F1 micro-precision    micro-recall 
     0.13175933      0.07136964      0.85643564 

3 enron Dataset

# PROPOSED EXERCISE
# Complete a similar work, for the popular Enron-Corpus of e-mails
# To create the corpus and the associated dataFrame
# A description of its labels appears in the following lists:
# https://bailando.berkeley.edu/enron/enron_categories.txt
# https://data.world/brianray/enron-email-dataset
# where for example "label 2.13" in the lists  is "label B.B13" for enron-labels
enron = enron()
Looking for datasetenronin the download directory

Looking for datasetenrononline...

Downloading datasetenron

trying URL 'https://cometa.ujaen.es/public/full/enron.rds'
Content type 'text/plain' length 227057 bytes (221 KB)
downloaded 221 KB
summary(enron)
enron$labels
enron <- remove_skewness_labels(enron, 10)
# Type "enron$" for its list of attributes
plot(enron, type = "LB")

# visual relations among labels
plot(enron, type = "LC")
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.

# create a holdout partition: train and predict to evaluate
ds <- create_holdout_partition(enron, c(train = 0.66, test = 0.34))
# an external GUI interface to explore the "enron" dataset
# mldrGUI() # press "escape" to exit GUI

Enron labels

Enron labelsets

Enron concurrence

3.1 Binary Relevance Naive Bayes

# Binary relevance ML strategy with naive Bayes base classifier
model_BR_NB <- utiml::br(ds$train, "NB")
predictionsBR <- predict(model_BR_NB, ds$test)
resultsBRPerExamples <- multilabel_evaluate(
    ds$test, predictionsBR,
    c("example-based")
)
resultsBRPerExamples
       accuracy              F1    hamming-loss       precision          recall 
    0.226812312     0.342346750     0.242906489     0.281762099     0.693615840 
subset-accuracy 
    0.001727116 
resultsBRPerLabel <- multilabel_evaluate(
    ds$test, predictionsBR,
    c("label-based")
)
resultsBRPerLabel
      macro-AUC        macro-F1 macro-precision    macro-recall       micro-AUC 
      0.6431262       0.1991138       0.2018842       0.4658723       0.7537622 
       micro-F1 micro-precision    micro-recall 
      0.3043222       0.1980077       0.6571719 

3.2 Classifier Chain Naive Bayes

# Classifier Chain ML strategy with naive Bayes base classifier
# Define the chain-order between labels: sample a random order
mychain <- sample(rownames(enron$labels))
model_CC_NB <- cc(ds$train, "NB", mychain)
predictionsCC <- predict(model_CC_NB, ds$test)
resultsCCPerExamples <- multilabel_evaluate(
    ds$test, predictionsCC,
    c("example-based")
)
resultsCCPerExamples
       accuracy              F1    hamming-loss       precision          recall 
    0.232044302     0.348937174     0.228678345     0.283524350     0.719364257 
subset-accuracy 
    0.003454231 
resultsCCPerLabel <- multilabel_evaluate(
    ds$test, predictionsCC,
    c("label-based")
)
resultsCCPerLabel
      macro-AUC        macro-F1 macro-precision    macro-recall       micro-AUC 
      0.6350099       0.2054387       0.2075607       0.4349603       0.7707133 
       micro-F1 micro-precision    micro-recall 
      0.3150634       0.2051323       0.6788747 

4 emotions Dataset

# "emotions" dataset to predict the set of emotions species
# that are present, given a ten-second audio clip:
# https://doi.org/10.1109/MLSP.2013.6661934
summary(emotions)
emotions$labels
emotions <- remove_skewness_labels(emotions, 10)
# Type "emotions$" for its list of attributes
plot(emotions, type = "LB")

# visual relations among labels
plot(emotions, type = "LC")
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.

# create a holdout partition: train and predict to evaluate
ds <- create_holdout_partition(emotions, c(train = 0.66, test = 0.34))
# an external GUI interface to explore the "emotions" dataset
# mldrGUI() # press "escape" to exit GUI

Emotions labels

Emotions labelsets

Emotions concurrence

4.1 Binary Relevance Naive Bayes

# Binary relevance ML strategy with naive Bayes base classifier
model_BR_NB <- utiml::br(ds$train, "NB")
predictionsBR <- predict(model_BR_NB, ds$test)
resultsBRPerExamples <- multilabel_evaluate(
    ds$test, predictionsBR,
    c("example-based")
)
resultsBRPerExamples
       accuracy              F1    hamming-loss       precision          recall 
      0.5045380       0.6050684       0.2739274       0.5594059       0.7351485 
subset-accuracy 
      0.1732673 
resultsBRPerLabel <- multilabel_evaluate(
    ds$test, predictionsBR,
    c("label-based")
)
resultsBRPerLabel
      macro-AUC        macro-F1 macro-precision    macro-recall       micro-AUC 
      0.7765587       0.6232502       0.5503731       0.7237345       0.7864504 
       micro-F1 micro-precision    micro-recall 
      0.6261261       0.5504950       0.7258486 

4.2 Classifier Chain Naive Bayes

# Classifier Chain ML strategy with naive Bayes base classifier
# Define the chain-order between labels: sample a random order
mychain <- sample(rownames(emotions$labels))
model_CC_NB <- cc(ds$train, "NB", mychain)
predictionsCC <- predict(model_CC_NB, ds$test)
resultsCCPerExamples <- multilabel_evaluate(
    ds$test, predictionsCC,
    c("example-based")
)
resultsCCPerExamples
       accuracy              F1    hamming-loss       precision          recall 
      0.5153465       0.6146393       0.2656766       0.5730198       0.7400990 
subset-accuracy 
      0.1782178 
resultsCCPerLabel <- multilabel_evaluate(
    ds$test, predictionsCC,
    c("label-based")
)
resultsCCPerLabel
      macro-AUC        macro-F1 macro-precision    macro-recall       micro-AUC 
      0.7794148       0.6292523       0.5617664       0.7229584       0.7890771 
       micro-F1 micro-precision    micro-recall 
      0.6332574       0.5616162       0.7258486 

5 genbase Dataset

# "genbase" dataset to predict the set of genbase species
# that are present, given a ten-second audio clip:
# https://doi.org/10.1109/MLSP.2013.6661934
summary(genbase)
genbase$labels
genbase <- remove_skewness_labels(genbase, 10)
# Type "genbase$" for its list of attributes
plot(genbase, type = "LB")

# visual relations among labels
plot(genbase, type = "LC")
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.
`major.tick.percentage` is not used any more, please directly use argument `major.tick.length`.

# create a holdout partition: train and predict to evaluate
ds <- create_holdout_partition(genbase, c(train = 0.66, test = 0.34))
# an external GUI interface to explore the "genbase" dataset
# mldrGUI() # press "escape" to exit GUI

Genbase labels

Genbase labelsets

Genbase concurrence

5.1 Binary Relevance Naive Bayes

# Binary relevance ML strategy with naive Bayes base classifier
model_BR_NB <- utiml::br(ds$train, "NB")
predictionsBR <- predict(model_BR_NB, ds$test)
resultsBRPerExamples <- multilabel_evaluate(
    ds$test, predictionsBR,
    c("example-based")
)
resultsBRPerExamples
       accuracy              F1    hamming-loss       precision          recall 
      0.9622222       0.9751111       0.0050000       0.9622222       1.0000000 
subset-accuracy 
      0.9200000 
resultsBRPerLabel <- multilabel_evaluate(
    ds$test, predictionsBR,
    c("label-based")
)
resultsBRPerLabel
      macro-AUC        macro-F1 macro-precision    macro-recall       micro-AUC 
      1.0000000       0.9322917       0.9007353       1.0000000       0.9991798 
       micro-F1 micro-precision    micro-recall 
      0.9683099       0.9385666       1.0000000 

5.2 Classifier Chain Naive Bayes

# Classifier Chain ML strategy with naive Bayes base classifier
# Define the chain-order between labels: sample a random order
mychain <- sample(rownames(genbase$labels))
model_CC_NB <- cc(ds$train, "NB", mychain)
predictionsCC <- predict(model_CC_NB, ds$test)
resultsCCPerExamples <- multilabel_evaluate(
    ds$test, predictionsCC,
    c("example-based")
)
resultsCCPerExamples
       accuracy              F1    hamming-loss       precision          recall 
    0.954074074     0.969777778     0.006111111     0.954074074     1.000000000 
subset-accuracy 
    0.902222222 
resultsCCPerLabel <- multilabel_evaluate(
    ds$test, predictionsCC,
    c("label-based")
)
resultsCCPerLabel
      macro-AUC        macro-F1 macro-precision    macro-recall       micro-AUC 
      0.9997185       0.9211806       0.8890595       1.0000000       0.9986942 
       micro-F1 micro-precision    micro-recall 
      0.9615385       0.9259259       1.0000000 
LS0tDQp0aXRsZTogIk11bHRpLUxhYmVsIENsYXNzaWZpY2F0aW9uIGluIFIiDQpvdXRwdXQ6DQogIGh0bWxfbm90ZWJvb2s6IA0KICAgIHRvYzogeWVzDQogICAgdG9jX2Zsb2F0OiB5ZXMNCiAgICBudW1iZXJfc2VjdGlvbnM6IHllcw0KLS0tDQoNCmBgYHtyfQ0KIyAidXRpbWwiIGFuZCAibWxkciIgcGFja2FnZXMgZm9yIG11bHRpLWxhYmVsIGNsYXNzaWZpY2F0aW9uIGluIFINCiMgaHR0cHM6Ly9qb3VybmFsLnItcHJvamVjdC5vcmcvYXJjaGl2ZS8yMDE4L1JKLTIwMTgtMDQxL1JKLTIwMTgtMDQxLnBkZg0KIyBodHRwczovL2dpdGh1Yi5jb20vcml2b2xsaS91dGltbA0KIyBodHRwczovL2NyYW4uci1wcm9qZWN0Lm9yZy93ZWIvcGFja2FnZXMvbWxkci92aWduZXR0ZXMvbWxkci5wZGYNCiMgaHR0cHM6Ly9naXRodWIuY29tL2ZjaGFydGUvbWxkcg0KbGlicmFyeSh1dGltbCkNCmxpYnJhcnkobWxkcikNCiMgZm9sbG93aW5nIHBhY2thZ2Ugb2ZmZXJzIGJlbmNobWFya3MgZm9yIG11bHRpLWxhYmVsIGNsYXNzaWZpY2F0aW9uDQpsaWJyYXJ5KG1sZHIuZGF0YXNldHMpDQpzZXQuc2VlZCgxMjMpDQpgYGANCg0KIyBuZzIwIERhdGFzZXQNCg0KYGBge3J9DQpzdW1tYXJ5KG5nMjApDQojIG5nMjAsIGEgY29ycHVzIHdpdGggMTkzMDAgZG9jdW1lbnRzLCAxMDA2IHdvcmRzIGFuZCAyMCBtdWx0aS1sYWJlbHMNCiMgS2VuIExhbmcsICJOZXdzd2VlZGVyOiBMZWFybmluZyB0byBmaWx0ZXIgbmV0bmV3cyIsIDEydGggSUNNTCBDb25mZXJlbmNlDQpuZzIwJGxhYmVscw0KIyBub3QgcHJhY3RpY2FsLCBidXQgcGFydCBvZiBpdHMgY29ycHVzIGNhbiBiZSB2aWV3ZWQgZG9pbmcNCm5nMjBjb3JwdXMgPC0gbmcyMCRkYXRhc2V0DQojIG5nMjAncyBiYWcgb2Ygd29yZHMNCmRpbShuZzIwY29ycHVzKQ0KIyBjb2xuYW1lcyhuZzIwY29ycHVzKQ0KIyBjb25zdWx0IHRoZSBoZWxwIG9mIHRoZSBmb2xsb3dpbmcgZnVuY3Rpb24NCm5nMjAgPC0gcmVtb3ZlX3NrZXduZXNzX2xhYmVscyhuZzIwLCAxMCkNCiMgbGFiZWwgYmF0IHBsb3QNCnBsb3QobmcyMCwgdHlwZSA9ICJMQiIpDQojIHZpc3VhbCByZWxhdGlvbnMgYW1vbmcgbGFiZWxzDQpwbG90KG5nMjAsIHR5cGUgPSAiTEMiKQ0KIyBjcmVhdGUgYSBob2xkb3V0IHBhcnRpdGlvbjogdHJhaW4gYW5kIHByZWRpY3QgdG8gZXZhbHVhdGUNCiMgSSBjcmVhdGVkIGEgc21hbGwgdGVzdCBwYXJ0aXRpb24gYXMgdGhlIHByZWRpY3Rpb24gc3RlcCB0YWtlcyBhIGxvbmcgdGltZQ0KZHMgPC0gY3JlYXRlX2hvbGRvdXRfcGFydGl0aW9uKG5nMjAsIGModHJhaW4gPSAwLjkwLCB0ZXN0ID0gMC4xMCkpDQpgYGANCg0KYGBge3J9DQojIGFuIGV4dGVybmFsIEdVSSBpbnRlcmZhY2UgdG8gZXhwbG9yZSB0aGUgIm5nMjAiIGRhdGFzZXQNCiMgbWxkckdVSSgpICMgcHJlc3MgImVzY2FwZSIgdG8gZXhpdCBHVUkNCmBgYA0KDQohW05HMjAgbGFiZWxzXSguLi9pbWFnZXMvbmcyMF9sYWJlbHMucG5nKQ0KDQohW05HMjAgbGFiZWxzZXRzXSguLi9pbWFnZXMvbmcyMF9sYWJlbHNldHMucG5nKQ0KDQohW05HMjAgY29uY3VycmVuY2VdKC4uL2ltYWdlcy9uZzIwX2NvbmN1cnJlbmNlLnBuZykNCg0KIyMgQmluYXJ5IFJlbGV2YW5jZSBOYWl2ZSBCYXllcw0KDQpgYGB7cn0NCiMgQmluYXJ5IHJlbGV2YW5jZSBNTCBzdHJhdGVneSB3aXRoIG5haXZlIEJheWVzIGJhc2UgY2xhc3NpZmllcg0KbW9kZWxfQlJfTkIgPC0gdXRpbWw6OmJyKGRzJHRyYWluLCAiTkIiKQ0KcHJlZGljdGlvbnNCUiA8LSBwcmVkaWN0KG1vZGVsX0JSX05CLCBkcyR0ZXN0KQ0KcmVzdWx0c0JSUGVyRXhhbXBsZXMgPC0gbXVsdGlsYWJlbF9ldmFsdWF0ZSgNCiAgICBkcyR0ZXN0LCBwcmVkaWN0aW9uc0JSLA0KICAgIGMoImV4YW1wbGUtYmFzZWQiKQ0KKQ0KcmVzdWx0c0JSUGVyRXhhbXBsZXMNCnJlc3VsdHNCUlBlckxhYmVsIDwtIG11bHRpbGFiZWxfZXZhbHVhdGUoDQogICAgZHMkdGVzdCwgcHJlZGljdGlvbnNCUiwNCiAgICBjKCJsYWJlbC1iYXNlZCIpDQopDQpyZXN1bHRzQlJQZXJMYWJlbA0KYGBgDQoNCiMjIENsYXNzaWZpZXIgQ2hhaW4gTmFpdmUgQmF5ZXMNCg0KYGBge3J9DQojIENsYXNzaWZpZXIgQ2hhaW4gTUwgc3RyYXRlZ3kgd2l0aCBuYWl2ZSBCYXllcyBiYXNlIGNsYXNzaWZpZXINCiMgRGVmaW5lIHRoZSBjaGFpbi1vcmRlciBiZXR3ZWVuIGxhYmVsczogc2FtcGxlIGEgcmFuZG9tIG9yZGVyDQpteWNoYWluIDwtIHNhbXBsZShyb3duYW1lcyhuZzIwJGxhYmVscykpDQptb2RlbF9DQ19OQiA8LSBjYyhkcyR0cmFpbiwgIk5CIiwgbXljaGFpbikNCnByZWRpY3Rpb25zQ0MgPC0gcHJlZGljdChtb2RlbF9DQ19OQiwgZHMkdGVzdCkNCnJlc3VsdHNDQ1BlckV4YW1wbGVzIDwtIG11bHRpbGFiZWxfZXZhbHVhdGUoDQogICAgZHMkdGVzdCwgcHJlZGljdGlvbnNDQywNCiAgICBjKCJleGFtcGxlLWJhc2VkIikNCikNCnJlc3VsdHNDQ1BlckV4YW1wbGVzDQpyZXN1bHRzQ0NQZXJMYWJlbCA8LSBtdWx0aWxhYmVsX2V2YWx1YXRlKA0KICAgIGRzJHRlc3QsIHByZWRpY3Rpb25zQ0MsDQogICAgYygibGFiZWwtYmFzZWQiKQ0KKQ0KcmVzdWx0c0NDUGVyTGFiZWwNCmBgYA0KDQojIGJpcmRzIERhdGFzZXQNCg0KYGBge3J9DQojICJiaXJkcyIgZGF0YXNldCB0byBwcmVkaWN0IHRoZSBzZXQgb2YgYmlyZHMgc3BlY2llcw0KIyB0aGF0IGFyZSBwcmVzZW50LCBnaXZlbiBhIHRlbi1zZWNvbmQgYXVkaW8gY2xpcDoNCiMgaHR0cHM6Ly9kb2kub3JnLzEwLjExMDkvTUxTUC4yMDEzLjY2NjE5MzQNCnN1bW1hcnkoYmlyZHMpDQpiaXJkcyRsYWJlbHMNCmJpcmRzIDwtIHJlbW92ZV9za2V3bmVzc19sYWJlbHMoYmlyZHMsIDEwKQ0KIyBUeXBlICJiaXJkcyQiIGZvciBpdHMgbGlzdCBvZiBhdHRyaWJ1dGVzDQpwbG90KGJpcmRzLCB0eXBlID0gIkxCIikNCiMgdmlzdWFsIHJlbGF0aW9ucyBhbW9uZyBsYWJlbHMNCnBsb3QoYmlyZHMsIHR5cGUgPSAiTEMiKQ0KIyBjcmVhdGUgYSBob2xkb3V0IHBhcnRpdGlvbjogdHJhaW4gYW5kIHByZWRpY3QgdG8gZXZhbHVhdGUNCmRzIDwtIGNyZWF0ZV9ob2xkb3V0X3BhcnRpdGlvbihiaXJkcywgYyh0cmFpbiA9IDAuNjYsIHRlc3QgPSAwLjM0KSkNCmBgYA0KDQpgYGB7cn0NCiMgYW4gZXh0ZXJuYWwgR1VJIGludGVyZmFjZSB0byBleHBsb3JlIHRoZSAiYmlyZHMiIGRhdGFzZXQNCiMgbWxkckdVSSgpICMgcHJlc3MgImVzY2FwZSIgdG8gZXhpdCBHVUkNCmBgYA0KDQohW0JpcmRzIGxhYmVsc10oLi4vaW1hZ2VzL2JpcmRzX2xhYmVscy5wbmcpDQoNCiFbQmlyZHMgbGFiZWxzZXRzXSguLi9pbWFnZXMvYmlyZHNfbGFiZWxzZXRzLnBuZykNCg0KIVtCaXJkcyBjb25jdXJyZW5jZV0oLi4vaW1hZ2VzL2JpcmRzX2NvbmN1cnJlbmNlLnBuZykNCg0KIyMgQmluYXJ5IFJlbGV2YW5jZSBOYWl2ZSBCYXllcw0KDQpgYGB7cn0NCiMgQmluYXJ5IHJlbGV2YW5jZSBNTCBzdHJhdGVneSB3aXRoIG5haXZlIEJheWVzIGJhc2UgY2xhc3NpZmllcg0KbW9kZWxfQlJfTkIgPC0gdXRpbWw6OmJyKGRzJHRyYWluLCAiTkIiKQ0KcHJlZGljdGlvbnNCUiA8LSBwcmVkaWN0KG1vZGVsX0JSX05CLCBkcyR0ZXN0KQ0KcmVzdWx0c0JSUGVyRXhhbXBsZXMgPC0gbXVsdGlsYWJlbF9ldmFsdWF0ZSgNCiAgICBkcyR0ZXN0LCBwcmVkaWN0aW9uc0JSLA0KICAgIGMoImV4YW1wbGUtYmFzZWQiKQ0KKQ0KcmVzdWx0c0JSUGVyRXhhbXBsZXMNCnJlc3VsdHNCUlBlckxhYmVsIDwtIG11bHRpbGFiZWxfZXZhbHVhdGUoDQogICAgZHMkdGVzdCwgcHJlZGljdGlvbnNCUiwNCiAgICBjKCJsYWJlbC1iYXNlZCIpDQopDQpyZXN1bHRzQlJQZXJMYWJlbA0KYGBgDQoNCiMjIENsYXNzaWZpZXIgQ2hhaW4gTmFpdmUgQmF5ZXMNCg0KYGBge3J9CQ0KIyBDbGFzc2lmaWVyIENoYWluIE1MIHN0cmF0ZWd5IHdpdGggbmFpdmUgQmF5ZXMgYmFzZSBjbGFzc2lmaWVyDQojIERlZmluZSB0aGUgY2hhaW4tb3JkZXIgYmV0d2VlbiBsYWJlbHM6IHNhbXBsZSBhIHJhbmRvbSBvcmRlcg0KbXljaGFpbiA8LSBzYW1wbGUocm93bmFtZXMoYmlyZHMkbGFiZWxzKSkNCm1vZGVsX0NDX05CIDwtIGNjKGRzJHRyYWluLCAiTkIiLCBteWNoYWluKQ0KcHJlZGljdGlvbnNDQyA8LSBwcmVkaWN0KG1vZGVsX0NDX05CLCBkcyR0ZXN0KQ0KcmVzdWx0c0NDUGVyRXhhbXBsZXMgPC0gbXVsdGlsYWJlbF9ldmFsdWF0ZSgNCiAgICBkcyR0ZXN0LCBwcmVkaWN0aW9uc0NDLA0KICAgIGMoImV4YW1wbGUtYmFzZWQiKQ0KKQ0KcmVzdWx0c0NDUGVyRXhhbXBsZXMNCnJlc3VsdHNDQ1BlckxhYmVsIDwtIG11bHRpbGFiZWxfZXZhbHVhdGUoDQogICAgZHMkdGVzdCwgcHJlZGljdGlvbnNDQywNCiAgICBjKCJsYWJlbC1iYXNlZCIpDQopDQpyZXN1bHRzQ0NQZXJMYWJlbA0KYGBgDQoNCiMgZW5yb24gRGF0YXNldA0KDQpgYGB7cn0NCiMgUFJPUE9TRUQgRVhFUkNJU0UNCiMgQ29tcGxldGUgYSBzaW1pbGFyIHdvcmssIGZvciB0aGUgcG9wdWxhciBFbnJvbi1Db3JwdXMgb2YgZS1tYWlscw0KIyBUbyBjcmVhdGUgdGhlIGNvcnB1cyBhbmQgdGhlIGFzc29jaWF0ZWQgZGF0YUZyYW1lDQojIEEgZGVzY3JpcHRpb24gb2YgaXRzIGxhYmVscyBhcHBlYXJzIGluIHRoZSBmb2xsb3dpbmcgbGlzdHM6DQojIGh0dHBzOi8vYmFpbGFuZG8uYmVya2VsZXkuZWR1L2Vucm9uL2Vucm9uX2NhdGVnb3JpZXMudHh0DQojIGh0dHBzOi8vZGF0YS53b3JsZC9icmlhbnJheS9lbnJvbi1lbWFpbC1kYXRhc2V0DQojIHdoZXJlIGZvciBleGFtcGxlICJsYWJlbCAyLjEzIiBpbiB0aGUgbGlzdHMgIGlzICJsYWJlbCBCLkIxMyIgZm9yIGVucm9uLWxhYmVscw0KZW5yb24gPSBlbnJvbigpDQpzdW1tYXJ5KGVucm9uKQ0KZW5yb24kbGFiZWxzDQplbnJvbiA8LSByZW1vdmVfc2tld25lc3NfbGFiZWxzKGVucm9uLCAxMCkNCiMgVHlwZSAiZW5yb24kIiBmb3IgaXRzIGxpc3Qgb2YgYXR0cmlidXRlcw0KcGxvdChlbnJvbiwgdHlwZSA9ICJMQiIpDQojIHZpc3VhbCByZWxhdGlvbnMgYW1vbmcgbGFiZWxzDQpwbG90KGVucm9uLCB0eXBlID0gIkxDIikNCiMgY3JlYXRlIGEgaG9sZG91dCBwYXJ0aXRpb246IHRyYWluIGFuZCBwcmVkaWN0IHRvIGV2YWx1YXRlDQpkcyA8LSBjcmVhdGVfaG9sZG91dF9wYXJ0aXRpb24oZW5yb24sIGModHJhaW4gPSAwLjY2LCB0ZXN0ID0gMC4zNCkpDQpgYGANCg0KYGBge3J9DQojIGFuIGV4dGVybmFsIEdVSSBpbnRlcmZhY2UgdG8gZXhwbG9yZSB0aGUgImVucm9uIiBkYXRhc2V0DQojIG1sZHJHVUkoKSAjIHByZXNzICJlc2NhcGUiIHRvIGV4aXQgR1VJDQpgYGANCg0KIVtFbnJvbiBsYWJlbHNdKC4uL2ltYWdlcy9lbnJvbl9sYWJlbHMucG5nKQ0KDQohW0Vucm9uIGxhYmVsc2V0c10oLi4vaW1hZ2VzL2Vucm9uX2xhYmVsc2V0cy5wbmcpDQoNCiFbRW5yb24gY29uY3VycmVuY2VdKC4uL2ltYWdlcy9lbnJvbl9jb25jdXJyZW5jZS5wbmcpDQoNCiMjIEJpbmFyeSBSZWxldmFuY2UgTmFpdmUgQmF5ZXMNCg0KYGBge3J9DQojIEJpbmFyeSByZWxldmFuY2UgTUwgc3RyYXRlZ3kgd2l0aCBuYWl2ZSBCYXllcyBiYXNlIGNsYXNzaWZpZXINCm1vZGVsX0JSX05CIDwtIHV0aW1sOjpicihkcyR0cmFpbiwgIk5CIikNCnByZWRpY3Rpb25zQlIgPC0gcHJlZGljdChtb2RlbF9CUl9OQiwgZHMkdGVzdCkNCnJlc3VsdHNCUlBlckV4YW1wbGVzIDwtIG11bHRpbGFiZWxfZXZhbHVhdGUoDQogICAgZHMkdGVzdCwgcHJlZGljdGlvbnNCUiwNCiAgICBjKCJleGFtcGxlLWJhc2VkIikNCikNCnJlc3VsdHNCUlBlckV4YW1wbGVzDQpyZXN1bHRzQlJQZXJMYWJlbCA8LSBtdWx0aWxhYmVsX2V2YWx1YXRlKA0KICAgIGRzJHRlc3QsIHByZWRpY3Rpb25zQlIsDQogICAgYygibGFiZWwtYmFzZWQiKQ0KKQ0KcmVzdWx0c0JSUGVyTGFiZWwNCmBgYA0KDQojIyBDbGFzc2lmaWVyIENoYWluIE5haXZlIEJheWVzDQoNCmBgYHtyfQkNCiMgQ2xhc3NpZmllciBDaGFpbiBNTCBzdHJhdGVneSB3aXRoIG5haXZlIEJheWVzIGJhc2UgY2xhc3NpZmllcg0KIyBEZWZpbmUgdGhlIGNoYWluLW9yZGVyIGJldHdlZW4gbGFiZWxzOiBzYW1wbGUgYSByYW5kb20gb3JkZXINCm15Y2hhaW4gPC0gc2FtcGxlKHJvd25hbWVzKGVucm9uJGxhYmVscykpDQptb2RlbF9DQ19OQiA8LSBjYyhkcyR0cmFpbiwgIk5CIiwgbXljaGFpbikNCnByZWRpY3Rpb25zQ0MgPC0gcHJlZGljdChtb2RlbF9DQ19OQiwgZHMkdGVzdCkNCnJlc3VsdHNDQ1BlckV4YW1wbGVzIDwtIG11bHRpbGFiZWxfZXZhbHVhdGUoDQogICAgZHMkdGVzdCwgcHJlZGljdGlvbnNDQywNCiAgICBjKCJleGFtcGxlLWJhc2VkIikNCikNCnJlc3VsdHNDQ1BlckV4YW1wbGVzDQpyZXN1bHRzQ0NQZXJMYWJlbCA8LSBtdWx0aWxhYmVsX2V2YWx1YXRlKA0KICAgIGRzJHRlc3QsIHByZWRpY3Rpb25zQ0MsDQogICAgYygibGFiZWwtYmFzZWQiKQ0KKQ0KcmVzdWx0c0NDUGVyTGFiZWwNCmBgYA0KDQojIGVtb3Rpb25zIERhdGFzZXQNCg0KYGBge3J9DQojICJlbW90aW9ucyIgZGF0YXNldCB0byBwcmVkaWN0IHRoZSBzZXQgb2YgZW1vdGlvbnMgc3BlY2llcw0KIyB0aGF0IGFyZSBwcmVzZW50LCBnaXZlbiBhIHRlbi1zZWNvbmQgYXVkaW8gY2xpcDoNCiMgaHR0cHM6Ly9kb2kub3JnLzEwLjExMDkvTUxTUC4yMDEzLjY2NjE5MzQNCnN1bW1hcnkoZW1vdGlvbnMpDQplbW90aW9ucyRsYWJlbHMNCmVtb3Rpb25zIDwtIHJlbW92ZV9za2V3bmVzc19sYWJlbHMoZW1vdGlvbnMsIDEwKQ0KIyBUeXBlICJlbW90aW9ucyQiIGZvciBpdHMgbGlzdCBvZiBhdHRyaWJ1dGVzDQpwbG90KGVtb3Rpb25zLCB0eXBlID0gIkxCIikNCiMgdmlzdWFsIHJlbGF0aW9ucyBhbW9uZyBsYWJlbHMNCnBsb3QoZW1vdGlvbnMsIHR5cGUgPSAiTEMiKQ0KIyBjcmVhdGUgYSBob2xkb3V0IHBhcnRpdGlvbjogdHJhaW4gYW5kIHByZWRpY3QgdG8gZXZhbHVhdGUNCmRzIDwtIGNyZWF0ZV9ob2xkb3V0X3BhcnRpdGlvbihlbW90aW9ucywgYyh0cmFpbiA9IDAuNjYsIHRlc3QgPSAwLjM0KSkNCmBgYA0KDQpgYGB7cn0NCiMgYW4gZXh0ZXJuYWwgR1VJIGludGVyZmFjZSB0byBleHBsb3JlIHRoZSAiZW1vdGlvbnMiIGRhdGFzZXQNCiMgbWxkckdVSSgpICMgcHJlc3MgImVzY2FwZSIgdG8gZXhpdCBHVUkNCmBgYA0KDQohW0Vtb3Rpb25zIGxhYmVsc10oLi4vaW1hZ2VzL2Vtb3Rpb25zX2xhYmVscy5wbmcpDQoNCiFbRW1vdGlvbnMgbGFiZWxzZXRzXSguLi9pbWFnZXMvZW1vdGlvbnNfbGFiZWxzZXRzLnBuZykNCg0KIVtFbW90aW9ucyBjb25jdXJyZW5jZV0oLi4vaW1hZ2VzL2Vtb3Rpb25zX2NvbmN1cnJlbmNlLnBuZykNCg0KIyMgQmluYXJ5IFJlbGV2YW5jZSBOYWl2ZSBCYXllcw0KDQpgYGB7cn0NCiMgQmluYXJ5IHJlbGV2YW5jZSBNTCBzdHJhdGVneSB3aXRoIG5haXZlIEJheWVzIGJhc2UgY2xhc3NpZmllcg0KbW9kZWxfQlJfTkIgPC0gdXRpbWw6OmJyKGRzJHRyYWluLCAiTkIiKQ0KcHJlZGljdGlvbnNCUiA8LSBwcmVkaWN0KG1vZGVsX0JSX05CLCBkcyR0ZXN0KQ0KcmVzdWx0c0JSUGVyRXhhbXBsZXMgPC0gbXVsdGlsYWJlbF9ldmFsdWF0ZSgNCiAgICBkcyR0ZXN0LCBwcmVkaWN0aW9uc0JSLA0KICAgIGMoImV4YW1wbGUtYmFzZWQiKQ0KKQ0KcmVzdWx0c0JSUGVyRXhhbXBsZXMNCnJlc3VsdHNCUlBlckxhYmVsIDwtIG11bHRpbGFiZWxfZXZhbHVhdGUoDQogICAgZHMkdGVzdCwgcHJlZGljdGlvbnNCUiwNCiAgICBjKCJsYWJlbC1iYXNlZCIpDQopDQpyZXN1bHRzQlJQZXJMYWJlbA0KYGBgDQoNCiMjIENsYXNzaWZpZXIgQ2hhaW4gTmFpdmUgQmF5ZXMNCg0KYGBge3J9CQ0KIyBDbGFzc2lmaWVyIENoYWluIE1MIHN0cmF0ZWd5IHdpdGggbmFpdmUgQmF5ZXMgYmFzZSBjbGFzc2lmaWVyDQojIERlZmluZSB0aGUgY2hhaW4tb3JkZXIgYmV0d2VlbiBsYWJlbHM6IHNhbXBsZSBhIHJhbmRvbSBvcmRlcg0KbXljaGFpbiA8LSBzYW1wbGUocm93bmFtZXMoZW1vdGlvbnMkbGFiZWxzKSkNCm1vZGVsX0NDX05CIDwtIGNjKGRzJHRyYWluLCAiTkIiLCBteWNoYWluKQ0KcHJlZGljdGlvbnNDQyA8LSBwcmVkaWN0KG1vZGVsX0NDX05CLCBkcyR0ZXN0KQ0KcmVzdWx0c0NDUGVyRXhhbXBsZXMgPC0gbXVsdGlsYWJlbF9ldmFsdWF0ZSgNCiAgICBkcyR0ZXN0LCBwcmVkaWN0aW9uc0NDLA0KICAgIGMoImV4YW1wbGUtYmFzZWQiKQ0KKQ0KcmVzdWx0c0NDUGVyRXhhbXBsZXMNCnJlc3VsdHNDQ1BlckxhYmVsIDwtIG11bHRpbGFiZWxfZXZhbHVhdGUoDQogICAgZHMkdGVzdCwgcHJlZGljdGlvbnNDQywNCiAgICBjKCJsYWJlbC1iYXNlZCIpDQopDQpyZXN1bHRzQ0NQZXJMYWJlbA0KYGBgDQoNCiMgZ2VuYmFzZSBEYXRhc2V0DQoNCmBgYHtyfQ0KIyAiZ2VuYmFzZSIgZGF0YXNldCB0byBwcmVkaWN0IHRoZSBzZXQgb2YgZ2VuYmFzZSBzcGVjaWVzDQojIHRoYXQgYXJlIHByZXNlbnQsIGdpdmVuIGEgdGVuLXNlY29uZCBhdWRpbyBjbGlwOg0KIyBodHRwczovL2RvaS5vcmcvMTAuMTEwOS9NTFNQLjIwMTMuNjY2MTkzNA0Kc3VtbWFyeShnZW5iYXNlKQ0KZ2VuYmFzZSRsYWJlbHMNCmdlbmJhc2UgPC0gcmVtb3ZlX3NrZXduZXNzX2xhYmVscyhnZW5iYXNlLCAxMCkNCiMgVHlwZSAiZ2VuYmFzZSQiIGZvciBpdHMgbGlzdCBvZiBhdHRyaWJ1dGVzDQpwbG90KGdlbmJhc2UsIHR5cGUgPSAiTEIiKQ0KIyB2aXN1YWwgcmVsYXRpb25zIGFtb25nIGxhYmVscw0KcGxvdChnZW5iYXNlLCB0eXBlID0gIkxDIikNCiMgY3JlYXRlIGEgaG9sZG91dCBwYXJ0aXRpb246IHRyYWluIGFuZCBwcmVkaWN0IHRvIGV2YWx1YXRlDQpkcyA8LSBjcmVhdGVfaG9sZG91dF9wYXJ0aXRpb24oZ2VuYmFzZSwgYyh0cmFpbiA9IDAuNjYsIHRlc3QgPSAwLjM0KSkNCmBgYA0KDQpgYGB7cn0NCiMgYW4gZXh0ZXJuYWwgR1VJIGludGVyZmFjZSB0byBleHBsb3JlIHRoZSAiZ2VuYmFzZSIgZGF0YXNldA0KIyBtbGRyR1VJKCkgIyBwcmVzcyAiZXNjYXBlIiB0byBleGl0IEdVSQ0KYGBgDQoNCiFbR2VuYmFzZSBsYWJlbHNdKC4uL2ltYWdlcy9nZW5iYXNlX2xhYmVscy5wbmcpDQoNCiFbR2VuYmFzZSBsYWJlbHNldHNdKC4uL2ltYWdlcy9nZW5iYXNlX2xhYmVsc2V0cy5wbmcpDQoNCiFbR2VuYmFzZSBjb25jdXJyZW5jZV0oLi4vaW1hZ2VzL2dlbmJhc2VfY29uY3VycmVuY2UucG5nKQ0KDQojIyBCaW5hcnkgUmVsZXZhbmNlIE5haXZlIEJheWVzDQoNCmBgYHtyfQ0KIyBCaW5hcnkgcmVsZXZhbmNlIE1MIHN0cmF0ZWd5IHdpdGggbmFpdmUgQmF5ZXMgYmFzZSBjbGFzc2lmaWVyDQptb2RlbF9CUl9OQiA8LSB1dGltbDo6YnIoZHMkdHJhaW4sICJOQiIpDQpwcmVkaWN0aW9uc0JSIDwtIHByZWRpY3QobW9kZWxfQlJfTkIsIGRzJHRlc3QpDQpyZXN1bHRzQlJQZXJFeGFtcGxlcyA8LSBtdWx0aWxhYmVsX2V2YWx1YXRlKA0KICAgIGRzJHRlc3QsIHByZWRpY3Rpb25zQlIsDQogICAgYygiZXhhbXBsZS1iYXNlZCIpDQopDQpyZXN1bHRzQlJQZXJFeGFtcGxlcw0KcmVzdWx0c0JSUGVyTGFiZWwgPC0gbXVsdGlsYWJlbF9ldmFsdWF0ZSgNCiAgICBkcyR0ZXN0LCBwcmVkaWN0aW9uc0JSLA0KICAgIGMoImxhYmVsLWJhc2VkIikNCikNCnJlc3VsdHNCUlBlckxhYmVsDQpgYGANCg0KIyMgQ2xhc3NpZmllciBDaGFpbiBOYWl2ZSBCYXllcw0KDQpgYGB7cn0JDQojIENsYXNzaWZpZXIgQ2hhaW4gTUwgc3RyYXRlZ3kgd2l0aCBuYWl2ZSBCYXllcyBiYXNlIGNsYXNzaWZpZXINCiMgRGVmaW5lIHRoZSBjaGFpbi1vcmRlciBiZXR3ZWVuIGxhYmVsczogc2FtcGxlIGEgcmFuZG9tIG9yZGVyDQpteWNoYWluIDwtIHNhbXBsZShyb3duYW1lcyhnZW5iYXNlJGxhYmVscykpDQptb2RlbF9DQ19OQiA8LSBjYyhkcyR0cmFpbiwgIk5CIiwgbXljaGFpbikNCnByZWRpY3Rpb25zQ0MgPC0gcHJlZGljdChtb2RlbF9DQ19OQiwgZHMkdGVzdCkNCnJlc3VsdHNDQ1BlckV4YW1wbGVzIDwtIG11bHRpbGFiZWxfZXZhbHVhdGUoDQogICAgZHMkdGVzdCwgcHJlZGljdGlvbnNDQywNCiAgICBjKCJleGFtcGxlLWJhc2VkIikNCikNCnJlc3VsdHNDQ1BlckV4YW1wbGVzDQpyZXN1bHRzQ0NQZXJMYWJlbCA8LSBtdWx0aWxhYmVsX2V2YWx1YXRlKA0KICAgIGRzJHRlc3QsIHByZWRpY3Rpb25zQ0MsDQogICAgYygibGFiZWwtYmFzZWQiKQ0KKQ0KcmVzdWx0c0NDUGVyTGFiZWwNCmBgYA==