Load the required R packages (additional ones will be loaded as needed)

library(ggplot2)
library(dplyr)
library(tidyr)
#library(lubridate)

Exploratory data analysis

Load and explore logged events

events = read.csv("data/events.csv")
str(events)
'data.frame':   95626 obs. of  4 variables:
 $ user  : chr  "9d744e5bf" "91489f7a9" "278a75edf" "53d6ab60c" ...
 $ log   : chr  "Assignment: Final Project" "Assignment: Final Project" "Assignment: Final Project" "Assignment: Final Project" ...
 $ action: chr  "Assignment" "Assignment" "Assignment" "Assignment" ...
 $ ts    : chr  "2019-10-26 09:37:12" "2019-10-26 09:09:34" "2019-10-18 12:05:28" "2019-10-19 13:28:37" ...

Transform time stamp data into the format suitable for processing date and time data

events$ts <- as.POSIXct(events$ts)
events$ts[1:10]
 [1] "2019-10-26 09:37:12 CEST" "2019-10-26 09:09:34 CEST" "2019-10-18 12:05:28 CEST" "2019-10-19 13:28:37 CEST"
 [5] "2019-10-15 23:38:13 CEST" "2019-10-18 17:51:43 CEST" "2019-10-18 15:22:56 CEST" "2019-10-22 13:46:51 CEST"
 [9] "2019-10-15 14:58:17 CEST" "2019-10-19 13:28:38 CEST"

We can order the events, for each user, based on the time stamp

Note: we will be using R’s pipe notation (|>) to make the code easier to understand and follow

events |>
  arrange(user, ts) -> events

head(events)

Let’s start by examining the time range the data is available for. It should (roughly) coincide with the start and the end of the course

course_start <- min(events$ts)
print(course_start)
[1] "2019-09-09 14:08:01 CEST"
#print(wday(course_start, week_start = 1, label = T))
course_end <- max(events$ts)
print(course_end)
[1] "2019-10-27 19:27:41 CET"
#print(wday(course_end, week_start = 1, label = T))

The course length (in weeks):

difftime(course_end, course_start, units="week")
Time difference of 6.894808 weeks

Since we want to make predictions based on the first couple of weeks data, we need to add the week variable

events |>
  mutate(week = strftime(ts, "%V") |> as.integer(),
         current_week = week - min(week) + 1) |>
  select(-week) -> events

Check the distribution of action counts across the course weeks

table(events$current_week)

    1     2     3     4     5     6     7 
 9271 11471 13699 18115 21838 15742  5490 

Also in proportions

table(events$current_week) |> prop.table() |> round(digits = 3)

    1     2     3     4     5     6     7 
0.097 0.120 0.143 0.189 0.228 0.165 0.057 

Examine character variables that represent different types of actions and logged events

apply(events |> select(action, log), 
      2, 
      function(x) length(unique(x)))
action    log 
    12     80 

Let’s examine the actions

table(events$action)|> prop.table() |> round(digits = 3)

Applications   Assignment  Course_view       Ethics     Feedback      General   Group_work Instructions     La_types 
       0.008        0.077        0.264        0.011        0.033        0.035        0.342        0.068        0.020 
  Practicals       Social       Theory 
       0.105        0.023        0.014 

Some of these actions refer to individual course topics, that is, to the access to lecture materials on distinct course topics. These are: General, Applications, Theory, Ethics, Feedback, La_types. We will rename the actions to make the meaning clearer

course_topics <- c("General", "Applications", "Theory",  "Ethics", "Feedback", "La_types")

events |>
  mutate(action = ifelse(test = action %in% course_topics, 
                         yes = paste("Lecture", action, sep = "_"), 
                         no = action)) -> events
unique(events$action)
 [1] "Course_view"          "Instructions"         "Group_work"           "Lecture_General"      "Practicals"          
 [6] "Social"               "Lecture_La_types"     "Assignment"           "Lecture_Feedback"     "Lecture_Applications"
[11] "Lecture_Ethics"       "Lecture_Theory"      

Examine also the log column

table(events$log) |> as.data.frame() |> arrange(desc(Freq)) |> View()

Too many distinct values, we will leave this variable aside, at least for now

Load and examine grades data

grades <- read.csv("data/grades.csv")
str(grades)
'data.frame':   130 obs. of  2 variables:
 $ user : chr  "6eba3ff82" "05b604102" "111422ee7" "b4658c3a9" ...
 $ grade: num  2.63 4.67 9.24 0 8.24 ...

Examine the summary statistics and distribution of the final grade

summary(grades$grade)
   Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
  0.000   5.666   7.954   7.254   9.006  10.000 
ggplot(grades, aes(x = grade)) +
  geom_density() +
  labs(x = "Final grade", 
       title = "Distribution of the final grade") +
  scale_x_continuous(breaks = 1:10) +
  theme_minimal()

It is not normally distributed, but skewed towards higher grade values

Let’s add course_outcome as a binary variable indicating if a student had a good or weak course outcome. Students whose final grade is above 50th percentile (median) will be considered as having good course outcome (HIGH), the rest will be considered as having weak course outcome (LOW)

grades |>
  mutate(course_outcome = ifelse(test=grade > median(grade),
                                 yes = "High", no = "Low")) |>
  mutate(course_outcome=factor(course_outcome)) -> grades
table(grades$course_outcome)

High  Low 
  65   65 

This gives us a perfectly balanced data set for the outcome prediction (classification) task.

Features

Two groups of action-based features will be computed and used for prediction: (note: active days are days with at least one learning action)

Since the idea is to create prediction models based on different number of weeks data, we will also need to compute feature values for different number of course weeks. Thus, we will create functions that compute features based on the data for the given number of course weeks (the input parameter).

To compute features based on counts per day, we need to add the date variable

events$date = as.Date(events$ts)
  1. Start with the total number of each type of learning actions

Note: to avoid having too many features (as action counts), we will consider all actions related to access to the lecture materials on different topics as one kind of action (‘Lecture’)

actions_tot_count <- function(events_data) {
  events_data |>
    mutate(action = ifelse(startsWith(action, "Lecture"), yes = "Lecture", no = action)) |>
    group_by(user, action) |>
    count() |>
    pivot_wider(id_cols = user, 
                names_from = action, 
                names_prefix = "cnt_",
                values_from = n, 
                values_fill = 0)
}

Check the function with the data from the first two weeks of the course

events_2_weeks <- events %>% filter(current_week <= 2)
actions_tot_count(events_2_weeks)
  1. Next, compute average number of actions (of any type) per day
avg_actions_per_day = function(events_data) {
  events_data |>
    count(user, date) |>
    group_by(user) |>
    summarise(avg_daily = median(n))
}
avg_actions_per_day(events_2_weeks)
  1. Entropy of daily action counts

Entropy is a measure of disorder in a system. Here it is used as an indicator of regularity of learning: lower the entropy, higher is the regularity and vice versa. Note: A nice explanation of the intuition behind the formula of Shannon entropy is given in this video.

Since we want to compute entropy of daily action counts, we need to compute (approximate) the probability of action counts for each day. We will do that by taking the proportion of daily action counts with respect to the total action counts for the given student

entropy_of_action_counts = function(events_data) {
  events_data |>
    group_by(user) |>
    mutate(tot_action_count = n())|>
    ungroup() |>
    group_by(user, date) |>
    summarise(daily_action_count = n(),
              daily_action_prop = daily_action_count/tot_action_count) |>
    ungroup() |>
    distinct() |>
    group_by(user) |>
    summarise(entropy = -sum(daily_action_prop*log(daily_action_prop)))
}
entropy_of_action_counts(events_2_weeks)
`summarise()` has grouped output by 'user', 'date'. You can override using the `.groups` argument.
  1. Number of active days (= days with at least one learning action)
active_days_count = function(events_data) {
  events_data |>
    group_by(user) |>
    summarise(active_days_cnt = n_distinct(date))
}
active_days_count(events_2_weeks)
  1. Average time distance between two consecutive active days

Note: for student with only 1 active day, avg_aday_dist will be NA. To avoid losing students due to the missing value of this feature, we will replace NAs with a large number (e.g., 2 x max distance), thus indicating that a student rarely (if ever) got back to the course activities

avg_dist_active_days = function(events_data) {
  events_data |>
    group_by(user) |>
    distinct(date) |>
    arrange(date) |>
    mutate(prev_aday = lag(date)) |>
    mutate(aday_dist = ifelse(is.na(prev_aday),
                              yes = NA,
                              no = difftime(date, prev_aday, units = "days"))) |>
    summarise(avg_aday_dist = median(aday_dist, na.rm = TRUE)) |>
    ungroup() -> df

  max_aday_dist = max(df$avg_aday_dist, na.rm = TRUE)
  df |>
    mutate(avg_aday_dist = ifelse(is.na(avg_aday_dist),
                                  yes = 2*max_aday_dist,
                                  no = avg_aday_dist))
}
avg_dist_active_days(events_2_weeks) |> summary()
     user           avg_aday_dist   
 Length:128         Min.   : 1.000  
 Class :character   1st Qu.: 1.000  
 Mode  :character   Median : 1.500  
                    Mean   : 2.469  
                    3rd Qu.: 2.000  
                    Max.   :20.000  

Create feature set for 2 weeks of data and examine feature relevance

Create a function that will allow for creating a feature set for any (given) number of course weeks

create_feature_set = function(events_data) {
  f1 = actions_tot_count(events_data)
  f2 = avg_actions_per_day(events_data)
  f3 = entropy_of_action_counts(events_data)
  f4 = active_days_count(events_data)
  f5 = avg_dist_active_days(events_data)

  f1 |> 
    inner_join(f2, by='user') |>
    inner_join(f3, by='user') |>
    inner_join(f4, by='user') |>
    inner_join(f5, by='user') |>
    as.data.frame()
}

Create the feature set based on the first two weeks of data

w2_feature_set = create_feature_set(events_2_weeks)
`summarise()` has grouped output by 'user', 'date'. You can override using the `.groups` argument.
str(w2_feature_set)
'data.frame':   128 obs. of  12 variables:
 $ user            : chr  "00a05cc62" "042b07ba1" "046c35846" "05b604102" ...
 $ cnt_Course_view : int  31 40 15 14 21 92 65 96 45 77 ...
 $ cnt_Group_work  : int  18 45 1 1 32 86 31 79 27 123 ...
 $ cnt_Instructions: int  9 19 6 7 4 32 29 28 13 27 ...
 $ cnt_Lecture     : int  11 9 1 1 0 48 9 35 22 23 ...
 $ cnt_Practicals  : int  5 1 1 0 3 16 16 3 3 2 ...
 $ cnt_Social      : int  4 16 0 0 0 14 2 29 1 11 ...
 $ cnt_Assignment  : int  0 3 0 1 0 3 3 8 0 12 ...
 $ avg_daily       : num  10 28 12 12 26 20 25 29 11 8 ...
 $ entropy         : num  1.373 1.043 0.512 0.512 0.811 ...
 $ active_days_cnt : int  5 4 2 2 3 11 7 10 5 9 ...
 $ avg_aday_dist   : num  2 2 10 10 2 1 1.5 1 2 1 ...
summary(w2_feature_set)
     user           cnt_Course_view  cnt_Group_work   cnt_Instructions  cnt_Lecture    cnt_Practicals  
 Length:128         Min.   :  3.00   Min.   :  0.00   Min.   : 0.00    Min.   : 0.00   Min.   : 0.000  
 Class :character   1st Qu.: 20.50   1st Qu.: 20.00   1st Qu.:10.75    1st Qu.: 4.00   1st Qu.: 1.000  
 Mode  :character   Median : 42.50   Median : 32.00   Median :18.00    Median :15.00   Median : 3.000  
                    Mean   : 50.88   Mean   : 49.00   Mean   :21.16    Mean   :21.57   Mean   : 6.016  
                    3rd Qu.: 66.50   3rd Qu.: 66.25   3rd Qu.:29.00    3rd Qu.:34.00   3rd Qu.: 9.000  
                    Max.   :212.00   Max.   :205.00   Max.   :73.00    Max.   :80.00   Max.   :33.000  
   cnt_Social     cnt_Assignment     avg_daily        entropy      active_days_cnt  avg_aday_dist   
 Min.   : 0.000   Min.   : 0.000   Min.   : 8.00   Min.   :0.000   Min.   : 1.000   Min.   : 1.000  
 1st Qu.: 2.000   1st Qu.: 0.000   1st Qu.:11.00   1st Qu.:1.043   1st Qu.: 4.000   1st Qu.: 1.000  
 Median : 4.500   Median : 1.000   Median :17.00   Median :1.593   Median : 7.000   Median : 1.500  
 Mean   : 9.445   Mean   : 3.977   Mean   :18.12   Mean   :1.423   Mean   : 6.586   Mean   : 2.469  
 3rd Qu.:15.000   3rd Qu.: 4.000   3rd Qu.:23.00   3rd Qu.:1.890   3rd Qu.: 9.000   3rd Qu.: 2.000  
 Max.   :45.000   Max.   :20.000   Max.   :41.50   Max.   :2.316   Max.   :13.000   Max.   :20.000  

Add the outcome variable

w2_feature_set |>
  inner_join(grades |> select(user, course_outcome) ) -> w2_data
Joining, by = "user"

Examine the relevance of features for the prediction of the outcome variable

Let’s first see how we can do it for one variable

ggplot(w2_data, aes(x = cnt_Course_view, fill=course_outcome)) +
  geom_density(alpha=0.5) +
  theme_minimal()

Now, do for all at once

Note: the notation .data[[fn]] in the code below allow us to access column from the ‘current’ data frame (in this case, w2_data) with the name given as the input variable of the function (fn)

feature_names <- colnames(w2_feature_set |> select(-user))

lapply(feature_names,
       function(fn) {
         ggplot(w2_data, aes(x = .data[[fn]], fill=course_outcome)) +
           geom_density(alpha=0.5) +
           theme_minimal()
       })
[[1]]

[[2]]

[[3]]

[[4]]

[[5]]

[[6]]

[[7]]

[[8]]

[[9]]

[[10]]

[[11]]

The plots suggest that all features are potentially relevant for predicting the course outcome.

Predictive modeling

Load additional R packages required for model building and evaluation

library(caret)
library(rpart)

We will use decision tree (as implemented in the rpart package) as the classification method, and will build a couple of decision tree (DT) models, one for each of the first five weeks of the course. We will build each model using the optimal value of the cp hyper-parameter, identified through 10-fold cross-validation (as we did before).

We will evaluate the models using the same metrics used before: accuracy, precision, recall, F1

build_DT_model <- function(train_data) {
  
  cp_grid <- expand.grid(.cp = seq(0.001, 0.1, 0.005))
  
  ctrl <- trainControl(method = "CV", 
                       number = 10,
                       classProbs = TRUE,
                       summaryFunction = twoClassSummary)
  
  dt <- train(x = train_data |> select(-course_outcome),
              y = train_data$course_outcome,
              method = "rpart",
              metric = "ROC",
              tuneGrid = cp_grid,
              trControl = ctrl)
  
  dt$finalModel
}
get_evaluation_measures <- function(model, test_data) {
  
  predicted_vals <- predict(model, 
                            test_data |> select(-course_outcome),
                            type = 'class')
  actual_vals <- test_data$course_outcome
  
  cm <- table(actual_vals, predicted_vals)
  
  # low achievement in the course is considered the positive class
  TP <- cm[2,2]
  TN <- cm[1,1]
  FP <- cm[1,2]
  FN <- cm[2,1]

  accuracy = sum(diag(cm)) / sum(cm)
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  F1 <- (2 * precision * recall) / (precision + recall)
  
  c(Accuracy = accuracy, 
    Precision = precision, 
    Recall = recall, 
    F1 = F1)
  
}

Create (classification) models for predicting course outcome, based on progresively more weeks of events data

Starting from week 1, up to week 5, create predictive models and examine their performance

models <- list()
eval_measures <- list()

for(k in 1:5) {
  
  print(paste("Starting computations for week", k))
  
  week_k_events <- events |> filter(current_week <= k)
  ds <- create_feature_set(week_k_events)
  ds <- inner_join(ds, grades)
  
  set.seed(2023)
  train_indices <- createDataPartition(ds$course_outcome, 
                                       p = 0.8, list = FALSE)
  train_ds <- ds[train_indices,] |> select(-c(user, grade))
  test_ds <- ds[-train_indices,] |> select(-c(user, grade))

  dt <- build_DT_model(train_ds)
  eval_dt <- get_evaluation_measures(dt, test_ds)
  
  models[[k]] <- dt
  eval_measures[[k]] <- eval_dt
}
[1] "Starting computations for week 1"
[1] "Starting computations for week 2"
[1] "Starting computations for week 3"
[1] "Starting computations for week 4"
[1] "Starting computations for week 5"

Compare the models based on the evaluation measures

# transform the eval_measures list into a df
eval_df <- bind_rows(eval_measures)

# embellish the evaluation report by: 
# 1) adding the week column; 
# 2) rounding the metric values to 4 digits; 
# 3) rearanging the order of columns 
eval_df |>
  mutate(week = 1:5) |>
  mutate(across(Accuracy:F1, round, digits=4)) |>
  select(week, Accuracy:F1)

The results suggest that already early in this course, it is possible to fairly well predict students at risk of low performance.

Examine the importance of features in an early in the course model with good performance

models[[2]]$variable.importance |> 
  as.data.frame() |>
  rename(importance = `models[[2]]$variable.importance`)

Regularity of daily action counts and the frequency of accessing the information on the course homepage (course info, syllabus, news, …) are the most important features. Next come the engagement in group work as well as the level of the overall engagement in the course (number of active days and average number of actions per (active) day)

LS0tCnRpdGxlOiAiUHJlZGljdGl2ZSBtb2RlbGxpbmc6IHByZWRpY3RpbmcgY291cnNlIG91dGNvbWVzIGluIGEgYmxlbmRlZCBwb3N0Z3JhZHVhdGUgY291cnNlIgpvdXRwdXQ6IGh0bWxfbm90ZWJvb2sKLS0tCgpMb2FkIHRoZSByZXF1aXJlZCBSIHBhY2thZ2VzIChhZGRpdGlvbmFsIG9uZXMgd2lsbCBiZSBsb2FkZWQgYXMgbmVlZGVkKQpgYGB7ciBtZXNzYWdlPUZBTFNFfQpsaWJyYXJ5KGdncGxvdDIpCmxpYnJhcnkoZHBseXIpCmxpYnJhcnkodGlkeXIpCmBgYAoKIyMgRXhwbG9yYXRvcnkgZGF0YSBhbmFseXNpcwoKIyMjIExvYWQgYW5kIGV4cGxvcmUgbG9nZ2VkIGV2ZW50cwpgYGB7cn0KZXZlbnRzID0gcmVhZC5jc3YoImRhdGEvZXZlbnRzLmNzdiIpCmBgYAoKYGBge3J9CnN0cihldmVudHMpCmBgYAoKVHJhbnNmb3JtIHRpbWUgc3RhbXAgZGF0YSBpbnRvIHRoZSBmb3JtYXQgc3VpdGFibGUgZm9yIHByb2Nlc3NpbmcgZGF0ZSBhbmQgdGltZSBkYXRhCmBgYHtyfQpldmVudHMkdHMgPC0gYXMuUE9TSVhjdChldmVudHMkdHMpCmBgYAoKYGBge3J9CmV2ZW50cyR0c1sxOjEwXQpgYGAKCldlIGNhbiBvcmRlciB0aGUgZXZlbnRzLCBmb3IgZWFjaCB1c2VyLCBiYXNlZCBvbiB0aGUgdGltZSBzdGFtcCAKCk5vdGU6IHdlIHdpbGwgYmUgdXNpbmcgUidzIHBpcGUgbm90YXRpb24gKHw+KSB0byBtYWtlIHRoZSBjb2RlIGVhc2llciB0byB1bmRlcnN0YW5kIGFuZCBmb2xsb3cgCmBgYHtyfQpldmVudHMgfD4KICBhcnJhbmdlKHVzZXIsIHRzKSAtPiBldmVudHMKCmhlYWQoZXZlbnRzKQpgYGAKCkxldCdzIHN0YXJ0IGJ5IGV4YW1pbmluZyB0aGUgdGltZSByYW5nZSB0aGUgZGF0YSBpcyBhdmFpbGFibGUgZm9yLiBJdCBzaG91bGQgKHJvdWdobHkpIGNvaW5jaWRlIHdpdGggdGhlIHN0YXJ0IGFuZCB0aGUgZW5kIG9mIHRoZSBjb3Vyc2UKYGBge3J9CmNvdXJzZV9zdGFydCA8LSBtaW4oZXZlbnRzJHRzKQpwcmludChjb3Vyc2Vfc3RhcnQpCmBgYAoKYGBge3J9CmNvdXJzZV9lbmQgPC0gbWF4KGV2ZW50cyR0cykKcHJpbnQoY291cnNlX2VuZCkKYGBgCgpUaGUgY291cnNlIGxlbmd0aCAoaW4gd2Vla3MpOgpgYGB7cn0KZGlmZnRpbWUoY291cnNlX2VuZCwgY291cnNlX3N0YXJ0LCB1bml0cz0id2VlayIpCmBgYAoKU2luY2Ugd2Ugd2FudCB0byBtYWtlIHByZWRpY3Rpb25zIGJhc2VkIG9uIHRoZSBmaXJzdCBjb3VwbGUgb2Ygd2Vla3MgZGF0YSwgd2UgbmVlZCB0byBhZGQgdGhlIHdlZWsgdmFyaWFibGUgCmBgYHtyfQpldmVudHMgfD4KICBtdXRhdGUod2VlayA9IHN0cmZ0aW1lKHRzLCAiJVYiKSB8PiBhcy5pbnRlZ2VyKCksCiAgICAgICAgIGN1cnJlbnRfd2VlayA9IHdlZWsgLSBtaW4od2VlaykgKyAxKSB8PgogIHNlbGVjdCgtd2VlaykgLT4gZXZlbnRzCmBgYAoKQ2hlY2sgdGhlIGRpc3RyaWJ1dGlvbiBvZiBhY3Rpb24gY291bnRzIGFjcm9zcyB0aGUgY291cnNlIHdlZWtzCmBgYHtyfQp0YWJsZShldmVudHMkY3VycmVudF93ZWVrKQpgYGAKQWxzbyBpbiBwcm9wb3J0aW9ucwpgYGB7cn0KdGFibGUoZXZlbnRzJGN1cnJlbnRfd2VlaykgfD4gcHJvcC50YWJsZSgpIHw+IHJvdW5kKGRpZ2l0cyA9IDMpCmBgYAoKRXhhbWluZSBjaGFyYWN0ZXIgdmFyaWFibGVzIHRoYXQgcmVwcmVzZW50IGRpZmZlcmVudCB0eXBlcyBvZiBhY3Rpb25zIGFuZCBsb2dnZWQgZXZlbnRzCmBgYHtyfQphcHBseShldmVudHMgfD4gc2VsZWN0KGFjdGlvbiwgbG9nKSwgCiAgICAgIDIsIAogICAgICBmdW5jdGlvbih4KSBsZW5ndGgodW5pcXVlKHgpKSkKYGBgCgpMZXQncyBleGFtaW5lIHRoZSBhY3Rpb25zCmBgYHtyfQp0YWJsZShldmVudHMkYWN0aW9uKXw+IHByb3AudGFibGUoKSB8PiByb3VuZChkaWdpdHMgPSAzKQpgYGAKU29tZSBvZiB0aGVzZSBhY3Rpb25zIHJlZmVyIHRvIGluZGl2aWR1YWwgY291cnNlIHRvcGljcywgdGhhdCBpcywgdG8gdGhlIGFjY2VzcyB0byBsZWN0dXJlIG1hdGVyaWFscyBvbiBkaXN0aW5jdCBjb3Vyc2UgdG9waWNzLiBUaGVzZSBhcmU6CkdlbmVyYWwsIEFwcGxpY2F0aW9ucywgVGhlb3J5LCAgRXRoaWNzLCBGZWVkYmFjaywgTGFfdHlwZXMuIApXZSB3aWxsIHJlbmFtZSB0aGUgYWN0aW9ucyB0byBtYWtlIHRoZSBtZWFuaW5nIGNsZWFyZXIKYGBge3J9CmNvdXJzZV90b3BpY3MgPC0gYygiR2VuZXJhbCIsICJBcHBsaWNhdGlvbnMiLCAiVGhlb3J5IiwgICJFdGhpY3MiLCAiRmVlZGJhY2siLCAiTGFfdHlwZXMiKQoKZXZlbnRzIHw+CiAgbXV0YXRlKGFjdGlvbiA9IGlmZWxzZSh0ZXN0ID0gYWN0aW9uICVpbiUgY291cnNlX3RvcGljcywgCiAgICAgICAgICAgICAgICAgICAgICAgICB5ZXMgPSBwYXN0ZSgiTGVjdHVyZSIsIGFjdGlvbiwgc2VwID0gIl8iKSwgCiAgICAgICAgICAgICAgICAgICAgICAgICBubyA9IGFjdGlvbikpIC0+IGV2ZW50cwpgYGAKCmBgYHtyfQp1bmlxdWUoZXZlbnRzJGFjdGlvbikKYGBgCgpFeGFtaW5lIGFsc28gdGhlIGxvZyBjb2x1bW4KYGBge3J9CnRhYmxlKGV2ZW50cyRsb2cpIHw+IGFzLmRhdGEuZnJhbWUoKSB8PiBhcnJhbmdlKGRlc2MoRnJlcSkpIHw+IFZpZXcoKQpgYGAKVG9vIG1hbnkgZGlzdGluY3QgdmFsdWVzLCB3ZSB3aWxsIGxlYXZlIHRoaXMgdmFyaWFibGUgYXNpZGUsIGF0IGxlYXN0IGZvciBub3cKCiMjIyBMb2FkIGFuZCBleGFtaW5lIGdyYWRlcyBkYXRhCmBgYHtyfQpncmFkZXMgPC0gcmVhZC5jc3YoImRhdGEvZ3JhZGVzLmNzdiIpCmBgYAoKYGBge3J9CnN0cihncmFkZXMpCmBgYApFeGFtaW5lIHRoZSBzdW1tYXJ5IHN0YXRpc3RpY3MgYW5kIGRpc3RyaWJ1dGlvbiBvZiB0aGUgZmluYWwgZ3JhZGUKYGBge3J9CnN1bW1hcnkoZ3JhZGVzJGdyYWRlKQpgYGAKCmBgYHtyfQpnZ3Bsb3QoZ3JhZGVzLCBhZXMoeCA9IGdyYWRlKSkgKwogIGdlb21fZGVuc2l0eSgpICsKICBsYWJzKHggPSAiRmluYWwgZ3JhZGUiLCAKICAgICAgIHRpdGxlID0gIkRpc3RyaWJ1dGlvbiBvZiB0aGUgZmluYWwgZ3JhZGUiKSArCiAgc2NhbGVfeF9jb250aW51b3VzKGJyZWFrcyA9IDE6MTApICsKICB0aGVtZV9taW5pbWFsKCkKYGBgCkl0IGlzIG5vdCBub3JtYWxseSBkaXN0cmlidXRlZCwgYnV0IHNrZXdlZCB0b3dhcmRzIGhpZ2hlciBncmFkZSB2YWx1ZXMKCkxldCdzIGFkZCAqY291cnNlX291dGNvbWUqIGFzIGEgYmluYXJ5IHZhcmlhYmxlIGluZGljYXRpbmcgaWYgYSBzdHVkZW50IGhhZCBhIGdvb2Qgb3Igd2VhayBjb3Vyc2Ugb3V0Y29tZS4KU3R1ZGVudHMgd2hvc2UgZmluYWwgZ3JhZGUgaXMgYWJvdmUgNTB0aCBwZXJjZW50aWxlIChtZWRpYW4pIHdpbGwgYmUgY29uc2lkZXJlZCBhcyBoYXZpbmcgZ29vZCBjb3Vyc2Ugb3V0Y29tZSAoSElHSCksIHRoZSByZXN0IHdpbGwgYmUgY29uc2lkZXJlZCBhcyBoYXZpbmcgd2VhayBjb3Vyc2Ugb3V0Y29tZSAoTE9XKQpgYGB7cn0KZ3JhZGVzIHw+CiAgbXV0YXRlKGNvdXJzZV9vdXRjb21lID0gaWZlbHNlKHRlc3Q9Z3JhZGUgPiBtZWRpYW4oZ3JhZGUpLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB5ZXMgPSAiSGlnaCIsIG5vID0gIkxvdyIpKSB8PgogIG11dGF0ZShjb3Vyc2Vfb3V0Y29tZT1mYWN0b3IoY291cnNlX291dGNvbWUpKSAtPiBncmFkZXMKYGBgCgpgYGB7cn0KdGFibGUoZ3JhZGVzJGNvdXJzZV9vdXRjb21lKQpgYGAKVGhpcyBnaXZlcyB1cyBhIHBlcmZlY3RseSBiYWxhbmNlZCBkYXRhIHNldCBmb3IgdGhlIG91dGNvbWUgcHJlZGljdGlvbiAoY2xhc3NpZmljYXRpb24pIHRhc2suIAoKCiMjIEZlYXR1cmVzCgpUd28gZ3JvdXBzIG9mIGFjdGlvbi1iYXNlZCBmZWF0dXJlcyB3aWxsIGJlIGNvbXB1dGVkIGFuZCB1c2VkIGZvciBwcmVkaWN0aW9uOgoobm90ZTogYWN0aXZlIGRheXMgYXJlIGRheXMgd2l0aCBhdCBsZWFzdCBvbmUgbGVhcm5pbmcgYWN0aW9uKQoKKiBGZWF0dXJlcyBiYXNlZCBvbiBsZWFybmluZyBhY3Rpb24gY291bnRzOgoqKiBUb3RhbCBudW1iZXIgb2YgZWFjaCB0eXBlIG9mIGxlYXJuaW5nIGFjdGlvbnMgKGNvbnNpZGVyaW5nIGFjdGl2ZSBkYXlzIG9ubHkpCioqIEF2ZXJhZ2UgbnVtYmVyIG9mIGFjdGlvbnMgKG9mIGFueSB0eXBlKSBwZXIgZGF5IChjb25zaWRlcmluZyBhY3RpdmUgZGF5cyBvbmx5KQoqKiBFbnRyb3B5IG9mIGRhaWx5IGFjdGlvbiBjb3VudHMgKGNvbnNpZGVyaW5nIGFjdGl2ZSBkYXlzIG9ubHkpCgoqIEZlYXR1cmVzIGJhc2VkIG9uIG51bWJlciBvZiBhY3RpdmUgZGF5cwoqKiBOdW1iZXIgb2YgYWN0aXZlIGRheXMKKiogQXZlcmFnZSB0aW1lIGRpc3RhbmNlIGJldHdlZW4gdHdvIGNvbnNlY3V0aXZlIGFjdGl2ZSBkYXlzCgpTaW5jZSB0aGUgaWRlYSBpcyB0byBjcmVhdGUgcHJlZGljdGlvbiBtb2RlbHMgYmFzZWQgb24gZGlmZmVyZW50IG51bWJlciBvZiB3ZWVrcyBkYXRhLCB3ZSB3aWxsIGFsc28gbmVlZCB0byBjb21wdXRlIGZlYXR1cmUgdmFsdWVzIGZvciBkaWZmZXJlbnQgbnVtYmVyIG9mIGNvdXJzZSB3ZWVrcy4gVGh1cywgd2Ugd2lsbCBjcmVhdGUgZnVuY3Rpb25zIHRoYXQgY29tcHV0ZSBmZWF0dXJlcyBiYXNlZCBvbiB0aGUgZGF0YSBmb3IgdGhlIGdpdmVuIG51bWJlciBvZiBjb3Vyc2Ugd2Vla3MgKHRoZSBpbnB1dCBwYXJhbWV0ZXIpLiAKClRvIGNvbXB1dGUgZmVhdHVyZXMgYmFzZWQgb24gY291bnRzIHBlciBkYXksIHdlIG5lZWQgdG8gYWRkIHRoZSBkYXRlIHZhcmlhYmxlCmBgYHtyfQpldmVudHMkZGF0ZSA9IGFzLkRhdGUoZXZlbnRzJHRzKQpgYGAKCigxKSBTdGFydCB3aXRoIHRoZSB0b3RhbCBudW1iZXIgb2YgZWFjaCB0eXBlIG9mIGxlYXJuaW5nIGFjdGlvbnMgCgpOb3RlOiB0byBhdm9pZCBoYXZpbmcgdG9vIG1hbnkgZmVhdHVyZXMgKGFzIGFjdGlvbiBjb3VudHMpLCB3ZSB3aWxsIGNvbnNpZGVyIGFsbCBhY3Rpb25zIHJlbGF0ZWQgdG8gYWNjZXNzIHRvIHRoZSBsZWN0dXJlIG1hdGVyaWFscyBvbiBkaWZmZXJlbnQgdG9waWNzIGFzIG9uZSBraW5kIG9mIGFjdGlvbiAoJ0xlY3R1cmUnKQpgYGB7cn0KYWN0aW9uc190b3RfY291bnQgPC0gZnVuY3Rpb24oZXZlbnRzX2RhdGEpIHsKICBldmVudHNfZGF0YSB8PgogICAgbXV0YXRlKGFjdGlvbiA9IGlmZWxzZShzdGFydHNXaXRoKGFjdGlvbiwgIkxlY3R1cmUiKSwgeWVzID0gIkxlY3R1cmUiLCBubyA9IGFjdGlvbikpIHw+CiAgICBncm91cF9ieSh1c2VyLCBhY3Rpb24pIHw+CiAgICBjb3VudCgpIHw+CiAgICBwaXZvdF93aWRlcihpZF9jb2xzID0gdXNlciwgCiAgICAgICAgICAgICAgICBuYW1lc19mcm9tID0gYWN0aW9uLCAKICAgICAgICAgICAgICAgIG5hbWVzX3ByZWZpeCA9ICJjbnRfIiwKICAgICAgICAgICAgICAgIHZhbHVlc19mcm9tID0gbiwgCiAgICAgICAgICAgICAgICB2YWx1ZXNfZmlsbCA9IDApCn0KYGBgCgpDaGVjayB0aGUgZnVuY3Rpb24gd2l0aCB0aGUgZGF0YSBmcm9tIHRoZSBmaXJzdCB0d28gd2Vla3Mgb2YgdGhlIGNvdXJzZQpgYGB7cn0KZXZlbnRzXzJfd2Vla3MgPC0gZXZlbnRzICU+JSBmaWx0ZXIoY3VycmVudF93ZWVrIDw9IDIpCmFjdGlvbnNfdG90X2NvdW50KGV2ZW50c18yX3dlZWtzKQpgYGAKCigyKSBOZXh0LCBjb21wdXRlIGF2ZXJhZ2UgbnVtYmVyIG9mIGFjdGlvbnMgKG9mIGFueSB0eXBlKSBwZXIgZGF5CgpgYGB7cn0KYXZnX2FjdGlvbnNfcGVyX2RheSA9IGZ1bmN0aW9uKGV2ZW50c19kYXRhKSB7CiAgZXZlbnRzX2RhdGEgfD4KICAgIGNvdW50KHVzZXIsIGRhdGUpIHw+CiAgICBncm91cF9ieSh1c2VyKSB8PgogICAgc3VtbWFyaXNlKGF2Z19kYWlseSA9IG1lZGlhbihuKSkKfQpgYGAKCmBgYHtyfQphdmdfYWN0aW9uc19wZXJfZGF5KGV2ZW50c18yX3dlZWtzKQpgYGAKCigzKSBFbnRyb3B5IG9mIGRhaWx5IGFjdGlvbiBjb3VudHMKCkVudHJvcHkgaXMgYSBtZWFzdXJlIG9mIGRpc29yZGVyIGluIGEgc3lzdGVtLiBIZXJlIGl0IGlzIHVzZWQgYXMgYW4gaW5kaWNhdG9yIG9mIHJlZ3VsYXJpdHkgb2YgbGVhcm5pbmc6IGxvd2VyIHRoZSBlbnRyb3B5LCBoaWdoZXIgaXMgdGhlIHJlZ3VsYXJpdHkgYW5kIHZpY2UgdmVyc2EuIApOb3RlOiBBIG5pY2UgZXhwbGFuYXRpb24gb2YgdGhlIGludHVpdGlvbiBiZWhpbmQgdGhlIGZvcm11bGEgb2YgU2hhbm5vbiBlbnRyb3B5IGlzIGdpdmVuIGluIFt0aGlzIHZpZGVvXShodHRwczovL3d3dy55b3V0dWJlLmNvbS93YXRjaD92PTBHQ0dhdzBRT2hBKS4KClNpbmNlIHdlIHdhbnQgdG8gY29tcHV0ZSBlbnRyb3B5IG9mIGRhaWx5IGFjdGlvbiBjb3VudHMsIHdlIG5lZWQgdG8gY29tcHV0ZSAoYXBwcm94aW1hdGUpIHRoZSBwcm9iYWJpbGl0eSBvZiBhY3Rpb24gY291bnRzIGZvciBlYWNoIGRheS4gV2Ugd2lsbCBkbyB0aGF0IGJ5IHRha2luZyB0aGUgcHJvcG9ydGlvbiBvZiBkYWlseSBhY3Rpb24gY291bnRzIHdpdGggcmVzcGVjdCB0byB0aGUgdG90YWwgYWN0aW9uIGNvdW50cyBmb3IgdGhlIGdpdmVuIHN0dWRlbnQKYGBge3J9CmVudHJvcHlfb2ZfYWN0aW9uX2NvdW50cyA9IGZ1bmN0aW9uKGV2ZW50c19kYXRhKSB7CiAgZXZlbnRzX2RhdGEgfD4KICAgIGdyb3VwX2J5KHVzZXIpIHw+CiAgICBtdXRhdGUodG90X2FjdGlvbl9jb3VudCA9IG4oKSl8PgogICAgdW5ncm91cCgpIHw+CiAgICBncm91cF9ieSh1c2VyLCBkYXRlKSB8PgogICAgc3VtbWFyaXNlKGRhaWx5X2FjdGlvbl9jb3VudCA9IG4oKSwKICAgICAgICAgICAgICBkYWlseV9hY3Rpb25fcHJvcCA9IGRhaWx5X2FjdGlvbl9jb3VudC90b3RfYWN0aW9uX2NvdW50KSB8PgogICAgdW5ncm91cCgpIHw+CiAgICBkaXN0aW5jdCgpIHw+CiAgICBncm91cF9ieSh1c2VyKSB8PgogICAgc3VtbWFyaXNlKGVudHJvcHkgPSAtc3VtKGRhaWx5X2FjdGlvbl9wcm9wKmxvZyhkYWlseV9hY3Rpb25fcHJvcCkpKQp9CmBgYAoKYGBge3J9CmVudHJvcHlfb2ZfYWN0aW9uX2NvdW50cyhldmVudHNfMl93ZWVrcykKYGBgCgooNCkgTnVtYmVyIG9mIGFjdGl2ZSBkYXlzICg9IGRheXMgd2l0aCBhdCBsZWFzdCBvbmUgbGVhcm5pbmcgYWN0aW9uKQoKYGBge3J9CmFjdGl2ZV9kYXlzX2NvdW50ID0gZnVuY3Rpb24oZXZlbnRzX2RhdGEpIHsKICBldmVudHNfZGF0YSB8PgogICAgZ3JvdXBfYnkodXNlcikgfD4KICAgIHN1bW1hcmlzZShhY3RpdmVfZGF5c19jbnQgPSBuX2Rpc3RpbmN0KGRhdGUpKQp9CmBgYAoKYGBge3J9CmFjdGl2ZV9kYXlzX2NvdW50KGV2ZW50c18yX3dlZWtzKQpgYGAKCig1KSBBdmVyYWdlIHRpbWUgZGlzdGFuY2UgYmV0d2VlbiB0d28gY29uc2VjdXRpdmUgYWN0aXZlIGRheXMKCk5vdGU6IGZvciBzdHVkZW50IHdpdGggb25seSAxIGFjdGl2ZSBkYXksIGF2Z19hZGF5X2Rpc3Qgd2lsbCBiZSBOQS4gVG8gYXZvaWQgbG9zaW5nIHN0dWRlbnRzIGR1ZSB0byB0aGUgbWlzc2luZyB2YWx1ZSBvZiB0aGlzIGZlYXR1cmUsIHdlIHdpbGwgcmVwbGFjZSBOQXMgd2l0aCBhIGxhcmdlIG51bWJlciAoZS5nLiwgMiB4IG1heCBkaXN0YW5jZSksIHRodXMgaW5kaWNhdGluZyB0aGF0IGEgc3R1ZGVudCByYXJlbHkgKGlmIGV2ZXIpIGdvdCBiYWNrIHRvIHRoZSBjb3Vyc2UgYWN0aXZpdGllcwpgYGB7cn0KYXZnX2Rpc3RfYWN0aXZlX2RheXMgPSBmdW5jdGlvbihldmVudHNfZGF0YSkgewogIGV2ZW50c19kYXRhIHw+CiAgICBncm91cF9ieSh1c2VyKSB8PgogICAgZGlzdGluY3QoZGF0ZSkgfD4KICAgIGFycmFuZ2UoZGF0ZSkgfD4KICAgIG11dGF0ZShwcmV2X2FkYXkgPSBsYWcoZGF0ZSkpIHw+CiAgICBtdXRhdGUoYWRheV9kaXN0ID0gaWZlbHNlKGlzLm5hKHByZXZfYWRheSksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHllcyA9IE5BLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICBubyA9IGRpZmZ0aW1lKGRhdGUsIHByZXZfYWRheSwgdW5pdHMgPSAiZGF5cyIpKSkgfD4KICAgIHN1bW1hcmlzZShhdmdfYWRheV9kaXN0ID0gbWVkaWFuKGFkYXlfZGlzdCwgbmEucm0gPSBUUlVFKSkgfD4KICAgIHVuZ3JvdXAoKSAtPiBkZgoKICBtYXhfYWRheV9kaXN0ID0gbWF4KGRmJGF2Z19hZGF5X2Rpc3QsIG5hLnJtID0gVFJVRSkKICBkZiB8PgogICAgbXV0YXRlKGF2Z19hZGF5X2Rpc3QgPSBpZmVsc2UoaXMubmEoYXZnX2FkYXlfZGlzdCksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB5ZXMgPSAyKm1heF9hZGF5X2Rpc3QsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBubyA9IGF2Z19hZGF5X2Rpc3QpKQp9CmBgYAoKYGBge3J9CmF2Z19kaXN0X2FjdGl2ZV9kYXlzKGV2ZW50c18yX3dlZWtzKSB8PiBzdW1tYXJ5KCkKYGBgCgojIyMgQ3JlYXRlIGZlYXR1cmUgc2V0IGZvciAyIHdlZWtzIG9mIGRhdGEgYW5kIGV4YW1pbmUgZmVhdHVyZSByZWxldmFuY2UKCkNyZWF0ZSBhIGZ1bmN0aW9uIHRoYXQgd2lsbCBhbGxvdyBmb3IgY3JlYXRpbmcgYSBmZWF0dXJlIHNldCBmb3IgYW55IChnaXZlbikgbnVtYmVyIG9mIGNvdXJzZSB3ZWVrcyAKYGBge3J9CmNyZWF0ZV9mZWF0dXJlX3NldCA9IGZ1bmN0aW9uKGV2ZW50c19kYXRhKSB7CiAgZjEgPSBhY3Rpb25zX3RvdF9jb3VudChldmVudHNfZGF0YSkKICBmMiA9IGF2Z19hY3Rpb25zX3Blcl9kYXkoZXZlbnRzX2RhdGEpCiAgZjMgPSBlbnRyb3B5X29mX2FjdGlvbl9jb3VudHMoZXZlbnRzX2RhdGEpCiAgZjQgPSBhY3RpdmVfZGF5c19jb3VudChldmVudHNfZGF0YSkKICBmNSA9IGF2Z19kaXN0X2FjdGl2ZV9kYXlzKGV2ZW50c19kYXRhKQoKICBmMSB8PiAKICAgIGlubmVyX2pvaW4oZjIsIGJ5PSd1c2VyJykgfD4KICAgIGlubmVyX2pvaW4oZjMsIGJ5PSd1c2VyJykgfD4KICAgIGlubmVyX2pvaW4oZjQsIGJ5PSd1c2VyJykgfD4KICAgIGlubmVyX2pvaW4oZjUsIGJ5PSd1c2VyJykgfD4KICAgIGFzLmRhdGEuZnJhbWUoKQp9CmBgYAoKQ3JlYXRlIHRoZSBmZWF0dXJlIHNldCBiYXNlZCBvbiB0aGUgZmlyc3QgdHdvIHdlZWtzIG9mIGRhdGEKYGBge3J9CncyX2ZlYXR1cmVfc2V0ID0gY3JlYXRlX2ZlYXR1cmVfc2V0KGV2ZW50c18yX3dlZWtzKQoKc3RyKHcyX2ZlYXR1cmVfc2V0KQpgYGAKCmBgYHtyfQpzdW1tYXJ5KHcyX2ZlYXR1cmVfc2V0KQpgYGAKCkFkZCB0aGUgb3V0Y29tZSB2YXJpYWJsZQpgYGB7cn0KdzJfZmVhdHVyZV9zZXQgfD4KICBpbm5lcl9qb2luKGdyYWRlcyB8PiBzZWxlY3QodXNlciwgY291cnNlX291dGNvbWUpICkgLT4gdzJfZGF0YQpgYGAKCkV4YW1pbmUgdGhlIHJlbGV2YW5jZSBvZiBmZWF0dXJlcyBmb3IgdGhlIHByZWRpY3Rpb24gb2YgdGhlIG91dGNvbWUgdmFyaWFibGUKCkxldCdzIGZpcnN0IHNlZSBob3cgd2UgY2FuIGRvIGl0IGZvciBvbmUgdmFyaWFibGUgCmBgYHtyfQpnZ3Bsb3QodzJfZGF0YSwgYWVzKHggPSBjbnRfQ291cnNlX3ZpZXcsIGZpbGw9Y291cnNlX291dGNvbWUpKSArCiAgZ2VvbV9kZW5zaXR5KGFscGhhPTAuNSkgKwogIHRoZW1lX21pbmltYWwoKQpgYGAKCk5vdywgZG8gZm9yIGFsbCBhdCBvbmNlCgpOb3RlOiB0aGUgbm90YXRpb24gYC5kYXRhW1tmbl1dYCBpbiB0aGUgY29kZSBiZWxvdyBhbGxvdyB1cyB0byBhY2Nlc3MgY29sdW1uIGZyb20gdGhlICdjdXJyZW50JyBkYXRhIGZyYW1lIChpbiB0aGlzIGNhc2UsIGB3Ml9kYXRhYCkgd2l0aCB0aGUgbmFtZSBnaXZlbiBhcyB0aGUgaW5wdXQgdmFyaWFibGUgb2YgdGhlIGZ1bmN0aW9uIChgZm5gKSAKYGBge3J9CmZlYXR1cmVfbmFtZXMgPC0gY29sbmFtZXModzJfZmVhdHVyZV9zZXQgfD4gc2VsZWN0KC11c2VyKSkKCmxhcHBseShmZWF0dXJlX25hbWVzLAogICAgICAgZnVuY3Rpb24oZm4pIHsKICAgICAgICAgZ2dwbG90KHcyX2RhdGEsIGFlcyh4ID0gLmRhdGFbW2ZuXV0sIGZpbGw9Y291cnNlX291dGNvbWUpKSArCiAgICAgICAgICAgZ2VvbV9kZW5zaXR5KGFscGhhPTAuNSkgKwogICAgICAgICAgIHRoZW1lX21pbmltYWwoKQogICAgICAgfSkKYGBgCgpUaGUgcGxvdHMgc3VnZ2VzdCB0aGF0IGFsbCBmZWF0dXJlcyBhcmUgcG90ZW50aWFsbHkgcmVsZXZhbnQgZm9yIHByZWRpY3RpbmcgdGhlIGNvdXJzZSBvdXRjb21lLgoKCiMjIFByZWRpY3RpdmUgbW9kZWxpbmcKCkxvYWQgYWRkaXRpb25hbCBSIHBhY2thZ2VzIHJlcXVpcmVkIGZvciBtb2RlbCBidWlsZGluZyBhbmQgZXZhbHVhdGlvbiAKYGBge3IgbWVzc2FnZT1GQUxTRX0KbGlicmFyeShjYXJldCkKbGlicmFyeShycGFydCkKYGBgCgpXZSB3aWxsIHVzZSBkZWNpc2lvbiB0cmVlIChhcyBpbXBsZW1lbnRlZCBpbiB0aGUgcnBhcnQgcGFja2FnZSkgYXMgdGhlIGNsYXNzaWZpY2F0aW9uIG1ldGhvZCwgYW5kIHdpbGwgYnVpbGQgYSBjb3VwbGUgb2YgZGVjaXNpb24gdHJlZSAoRFQpIG1vZGVscywgb25lIGZvciBlYWNoIG9mIHRoZSBmaXJzdCBmaXZlIHdlZWtzIG9mIHRoZSBjb3Vyc2UuIFdlIHdpbGwgYnVpbGQgZWFjaCBtb2RlbCB1c2luZyB0aGUgb3B0aW1hbCB2YWx1ZSBvZiB0aGUgYGNwYCBoeXBlci1wYXJhbWV0ZXIsIGlkZW50aWZpZWQgdGhyb3VnaCAxMC1mb2xkIGNyb3NzLXZhbGlkYXRpb24gKGFzIHdlIGRpZCBiZWZvcmUpLiAKCldlIHdpbGwgZXZhbHVhdGUgdGhlIG1vZGVscyB1c2luZyB0aGUgc2FtZSBtZXRyaWNzIHVzZWQgYmVmb3JlOiBhY2N1cmFjeSwgcHJlY2lzaW9uLCByZWNhbGwsIEYxCmBgYHtyfQpidWlsZF9EVF9tb2RlbCA8LSBmdW5jdGlvbih0cmFpbl9kYXRhKSB7CiAgCiAgY3BfZ3JpZCA8LSBleHBhbmQuZ3JpZCguY3AgPSBzZXEoMC4wMDEsIDAuMSwgMC4wMDUpKQogIAogIGN0cmwgPC0gdHJhaW5Db250cm9sKG1ldGhvZCA9ICJDViIsIAogICAgICAgICAgICAgICAgICAgICAgIG51bWJlciA9IDEwLAogICAgICAgICAgICAgICAgICAgICAgIGNsYXNzUHJvYnMgPSBUUlVFLAogICAgICAgICAgICAgICAgICAgICAgIHN1bW1hcnlGdW5jdGlvbiA9IHR3b0NsYXNzU3VtbWFyeSkKICAKICBkdCA8LSB0cmFpbih4ID0gdHJhaW5fZGF0YSB8PiBzZWxlY3QoLWNvdXJzZV9vdXRjb21lKSwKICAgICAgICAgICAgICB5ID0gdHJhaW5fZGF0YSRjb3Vyc2Vfb3V0Y29tZSwKICAgICAgICAgICAgICBtZXRob2QgPSAicnBhcnQiLAogICAgICAgICAgICAgIG1ldHJpYyA9ICJST0MiLAogICAgICAgICAgICAgIHR1bmVHcmlkID0gY3BfZ3JpZCwKICAgICAgICAgICAgICB0ckNvbnRyb2wgPSBjdHJsKQogIAogIGR0JGZpbmFsTW9kZWwKfQpgYGAKCgpgYGB7cn0KZ2V0X2V2YWx1YXRpb25fbWVhc3VyZXMgPC0gZnVuY3Rpb24obW9kZWwsIHRlc3RfZGF0YSkgewogIAogIHByZWRpY3RlZF92YWxzIDwtIHByZWRpY3QobW9kZWwsIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgdGVzdF9kYXRhIHw+IHNlbGVjdCgtY291cnNlX291dGNvbWUpLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgdHlwZSA9ICdjbGFzcycpCiAgYWN0dWFsX3ZhbHMgPC0gdGVzdF9kYXRhJGNvdXJzZV9vdXRjb21lCiAgCiAgY20gPC0gdGFibGUoYWN0dWFsX3ZhbHMsIHByZWRpY3RlZF92YWxzKQogIAogICMgbG93IGFjaGlldmVtZW50IGluIHRoZSBjb3Vyc2UgaXMgY29uc2lkZXJlZCB0aGUgcG9zaXRpdmUgY2xhc3MKICBUUCA8LSBjbVsyLDJdCiAgVE4gPC0gY21bMSwxXQogIEZQIDwtIGNtWzEsMl0KICBGTiA8LSBjbVsyLDFdCgogIGFjY3VyYWN5ID0gc3VtKGRpYWcoY20pKSAvIHN1bShjbSkKICBwcmVjaXNpb24gPC0gVFAgLyAoVFAgKyBGUCkKICByZWNhbGwgPC0gVFAgLyAoVFAgKyBGTikKICBGMSA8LSAoMiAqIHByZWNpc2lvbiAqIHJlY2FsbCkgLyAocHJlY2lzaW9uICsgcmVjYWxsKQogIAogIGMoQWNjdXJhY3kgPSBhY2N1cmFjeSwgCiAgICBQcmVjaXNpb24gPSBwcmVjaXNpb24sIAogICAgUmVjYWxsID0gcmVjYWxsLCAKICAgIEYxID0gRjEpCiAgCn0KYGBgCgoKIyMjIENyZWF0ZSAoY2xhc3NpZmljYXRpb24pIG1vZGVscyBmb3IgcHJlZGljdGluZyBjb3Vyc2Ugb3V0Y29tZSwgYmFzZWQgb24gcHJvZ3Jlc2l2ZWx5IG1vcmUgd2Vla3Mgb2YgZXZlbnRzIGRhdGEKClN0YXJ0aW5nIGZyb20gd2VlayAxLCB1cCB0byB3ZWVrIDUsIGNyZWF0ZSBwcmVkaWN0aXZlIG1vZGVscyBhbmQgZXhhbWluZSB0aGVpciBwZXJmb3JtYW5jZQpgYGB7ciB3YXJuaW5nPUZBTFNFLCBtZXNzYWdlPUZBTFNFfQptb2RlbHMgPC0gbGlzdCgpCmV2YWxfbWVhc3VyZXMgPC0gbGlzdCgpCgpmb3IoayBpbiAxOjUpIHsKICAKICBwcmludChwYXN0ZSgiU3RhcnRpbmcgY29tcHV0YXRpb25zIGZvciB3ZWVrIiwgaykpCiAgCiAgd2Vla19rX2V2ZW50cyA8LSBldmVudHMgfD4gZmlsdGVyKGN1cnJlbnRfd2VlayA8PSBrKQogIGRzIDwtIGNyZWF0ZV9mZWF0dXJlX3NldCh3ZWVrX2tfZXZlbnRzKQogIGRzIDwtIGlubmVyX2pvaW4oZHMsIGdyYWRlcykKICAKICBzZXQuc2VlZCgyMDIzKQogIHRyYWluX2luZGljZXMgPC0gY3JlYXRlRGF0YVBhcnRpdGlvbihkcyRjb3Vyc2Vfb3V0Y29tZSwgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHAgPSAwLjgsIGxpc3QgPSBGQUxTRSkKICB0cmFpbl9kcyA8LSBkc1t0cmFpbl9pbmRpY2VzLF0gfD4gc2VsZWN0KC1jKHVzZXIsIGdyYWRlKSkKICB0ZXN0X2RzIDwtIGRzWy10cmFpbl9pbmRpY2VzLF0gfD4gc2VsZWN0KC1jKHVzZXIsIGdyYWRlKSkKCiAgZHQgPC0gYnVpbGRfRFRfbW9kZWwodHJhaW5fZHMpCiAgZXZhbF9kdCA8LSBnZXRfZXZhbHVhdGlvbl9tZWFzdXJlcyhkdCwgdGVzdF9kcykKICAKICBtb2RlbHNbW2tdXSA8LSBkdAogIGV2YWxfbWVhc3VyZXNbW2tdXSA8LSBldmFsX2R0Cn0KYGBgCgpDb21wYXJlIHRoZSBtb2RlbHMgYmFzZWQgb24gdGhlIGV2YWx1YXRpb24gbWVhc3VyZXMKYGBge3J9CiMgdHJhbnNmb3JtIHRoZSBldmFsX21lYXN1cmVzIGxpc3QgaW50byBhIGRmCmV2YWxfZGYgPC0gYmluZF9yb3dzKGV2YWxfbWVhc3VyZXMpCgojIGVtYmVsbGlzaCB0aGUgZXZhbHVhdGlvbiByZXBvcnQgYnk6IAojIDEpIGFkZGluZyB0aGUgd2VlayBjb2x1bW47IAojIDIpIHJvdW5kaW5nIHRoZSBtZXRyaWMgdmFsdWVzIHRvIDQgZGlnaXRzOyAKIyAzKSByZWFyYW5naW5nIHRoZSBvcmRlciBvZiBjb2x1bW5zIApldmFsX2RmIHw+CiAgbXV0YXRlKHdlZWsgPSAxOjUpIHw+CiAgbXV0YXRlKGFjcm9zcyhBY2N1cmFjeTpGMSwgcm91bmQsIGRpZ2l0cz00KSkgfD4KICBzZWxlY3Qod2VlaywgQWNjdXJhY3k6RjEpCmBgYApUaGUgcmVzdWx0cyBzdWdnZXN0IHRoYXQgYWxyZWFkeSBlYXJseSBpbiB0aGlzIGNvdXJzZSwgaXQgaXMgcG9zc2libGUgdG8gZmFpcmx5IHdlbGwgcHJlZGljdCBzdHVkZW50cyBhdCByaXNrIG9mIGxvdyBwZXJmb3JtYW5jZS4KCkV4YW1pbmUgdGhlIGltcG9ydGFuY2Ugb2YgZmVhdHVyZXMgaW4gYW4gZWFybHkgaW4gdGhlIGNvdXJzZSBtb2RlbCB3aXRoIGdvb2QgcGVyZm9ybWFuY2UKYGBge3J9Cm1vZGVsc1tbMl1dJHZhcmlhYmxlLmltcG9ydGFuY2UgfD4gCiAgYXMuZGF0YS5mcmFtZSgpIHw+CiAgcmVuYW1lKGltcG9ydGFuY2UgPSBgbW9kZWxzW1syXV0kdmFyaWFibGUuaW1wb3J0YW5jZWApCmBgYApSZWd1bGFyaXR5IG9mIGRhaWx5IGFjdGlvbiBjb3VudHMgYW5kIHRoZSBmcmVxdWVuY3kgb2YgYWNjZXNzaW5nIHRoZSBpbmZvcm1hdGlvbiBvbiB0aGUgY291cnNlIGhvbWVwYWdlIChjb3Vyc2UgaW5mbywgc3lsbGFidXMsIG5ld3MsIC4uLikgYXJlIHRoZSBtb3N0IGltcG9ydGFudCBmZWF0dXJlcy4gTmV4dCBjb21lIHRoZSBlbmdhZ2VtZW50IGluIGdyb3VwIHdvcmsgYXMgd2VsbCBhcyB0aGUgbGV2ZWwgb2YgdGhlIG92ZXJhbGwgZW5nYWdlbWVudCBpbiB0aGUgY291cnNlIChudW1iZXIgb2YgYWN0aXZlIGRheXMgYW5kIGF2ZXJhZ2UgbnVtYmVyIG9mIGFjdGlvbnMgcGVyIChhY3RpdmUpIGRheSk=