diff --git a/.gitmodules b/.gitmodules index 0909ae9..7502afb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "bibsp"] path = bibsp url = https://github.com/dbosk/bibsp.git +[submodule "didactic"] + path = didactic + url = https://github.com/dbosk/didactic.git diff --git a/Makefile b/Makefile index d1ed36c..f5e3a4e 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,3 @@ -SUBDIR= course-design SUBDIR+= modules INCLUDE_MAKEFILES?= ./makefiles diff --git a/adm/grading/.gitignore b/adm/grading/.gitignore new file mode 100644 index 0000000..54d71cd --- /dev/null +++ b/adm/grading/.gitignore @@ -0,0 +1,8 @@ +_minted-vetcyb-grading/ +test-grade_reflections.txt +test-grade_reflections-err.txt +vetcyb-grading.out +vetcyb-grading.pdf +vetcyb-grading.sh +vetcyb-grading.toc +vetcyb24p2.reflections.d/ diff --git a/adm/grading/Makefile b/adm/grading/Makefile new file mode 100644 index 0000000..3c91f61 --- /dev/null +++ b/adm/grading/Makefile @@ -0,0 +1,35 @@ +NOWEAVEFLAGS=-delay +LATEXFLAGS=-shell-escape + +.PHONY: all +all: vetcyb-grading.pdf vetcyb-grading.sh + +.INTERMEDIATE: vetcyb-grading.tex +vetcyb-grading.pdf: vetcyb-grading.tex +vetcyb-grading.sh: vetcyb-grading.sh.nw + +vetcyb-grading.pdf: prompts.toml + +GRADE_RESULTS=test-grade_reflections-err.txt + +vetcyb-grading.pdf: ${GRADE_RESULTS} +vetcyb-grading.pdf: ./instruction-qualitative.md ./submission-qualitative.md + +${GRADE_RESULTS}: test-grade-reflections.sh + bash $< + +test-grade-reflections.sh: vetcyb-grading.sh.nw vetcyb-grading.sh + ${NOTANGLE.sh} + +.PHONY: clean +clean: + ${RM} vetcyb-grading.pdf vetcyb-grading.sh + ${RM} *.aux *.fdb_* *.fls *.log *.pdf *.tex + ${RM} test-grade-reflections.sh + ${RM} test-grade_reflections.txt + ${RM} test-grade_reflections-err.txt + + +INCLUDE_MAKEFILES=../../makefiles +include ${INCLUDE_MAKEFILES}/noweb.mk +include ${INCLUDE_MAKEFILES}/tex.mk diff --git a/adm/grading/grade_seminar.sh b/adm/grading/grade_seminar.sh new file mode 100644 index 0000000..c122c78 --- /dev/null +++ b/adm/grading/grade_seminar.sh @@ -0,0 +1,45 @@ +oldIFS=$IFS +IFS=$'\n' + +course="${1}" +seminar_date="${2}" +file="${3}" + +teachers="(dbosk|masvanbe|elzen|gylje)@kth.se" + +get_grade_data() { + tail -n +2 ${file} \ + | grep -Ev "${teachers}" \ + | cut -d, -f 2,3,5 \ + > participation-seminar.csv + + for user in $(cat participation-seminar.csv | cut -d, -f 1 | sort -u) + do + echo -n "$user "; + sum_minutes=0; + for minutes in $(grep $user participation-seminar.csv | cut -d, -f 3) + do + sum_minutes=$(($sum_minutes + $minutes)); + done; + echo $sum_minutes; + done +} + +open ${file} & +get_grade_data +read -p "Enter the limit in minutes: " limit_minutes + +for result in $(get_grade_data) +do + user=$(echo $result | cut -d ' ' -f 1); + minutes=$(echo $result | cut -d ' ' -f 2); + if [ $minutes -ge $limit_minutes ] + then + canvaslms grade -c ${course} -a "Live seminar ${seminar_date}" \ + -u $user -g complete + else + canvaslms grade -c ${course} -a "Live seminar ${seminar_date}" \ + -u $user -g incomplete \ + -m "You had $minutes minutes of participation out of more than $limit_minutes minutes." + fi +done diff --git a/adm/grading/instruction-qualitative.md b/adm/grading/instruction-qualitative.md new file mode 100644 index 0000000..b741528 --- /dev/null +++ b/adm/grading/instruction-qualitative.md @@ -0,0 +1,24 @@ + +# Reflection on qualitative methods + +## Metadata + +- Unlocks: None +- Due: None +- Locks: None +- Ungraded submissions: 28 +- Submission type: ['online_text_entry', 'online_upload'] +- URL: https://canvas.kth.se/courses/50613/assignments/318671 +- Submissions: https://canvas.kth.se/courses/50613/assignments/318671/submissions?zip=1 + +## Instruction + +What do qualitative methods contribute to security? How do they +complement the other methods that we\'ve talked about? + +To help you think about this, return to [How do you know it\'s secure? +Passwords](https://canvas.kth.se/courses/50613/assignments/315624 "How do you know it's secure? Passwords"). +How did the different methods fit together there? + + + diff --git a/adm/grading/instruction-statistics.md b/adm/grading/instruction-statistics.md new file mode 100644 index 0000000..38ab9d3 --- /dev/null +++ b/adm/grading/instruction-statistics.md @@ -0,0 +1,23 @@ + +# Reflection on the use of statistics + +## Metadata + +- Unlocks: None +- Due: None +- Locks: None +- Ungraded submissions: 27 +- Submission type: ['online_text_entry', 'online_upload'] +- URL: https://canvas.kth.se/courses/50613/assignments/318595 +- Submissions: https://canvas.kth.se/courses/50613/assignments/318595/submissions?zip=1 + +## Instruction + +In what ways do we use statistics in security research? Consider how the +papers you\'ve read so far have used statistics. + +What about machine learning? (Machine learning is basically black-box +automated statistics.) + + + diff --git a/adm/grading/prompts.toml b/adm/grading/prompts.toml new file mode 100644 index 0000000..6f6cfcf --- /dev/null +++ b/adm/grading/prompts.toml @@ -0,0 +1,36 @@ +# Prompt config files +# more details and examples at https://github.com/efugier/smartcat#configuration + +[empty] +api = "openai" +messages = [] +char_limit = 100000 + +[default] +api = "openai" +model = "gpt-4o" +temperature = 1.0 +char_limit = 100000 + +[[default.messages]] +role = "system" +content = "You are an extremely skilled programmer and scientist with a keen eye for detail and an emphasis on readable code. You have been tasked with acting as a smart version of the cat unix program. You take text and a prompt in and write text out. For that reason, it is of crucial importance to just write the desired output. Do not under any circumstance write any comment or thought as your output will be piped into other programs. Do not write the markdown delimiters for code as well. Sometimes you will be asked to implement or extend some input code. Same thing goes here, write only what was asked because what you write will be directly added to the user's editor. Never ever write ``` around the code. Make sure to keep the indentation and formatting. " + +[4o] +api = "openai" +model = "gpt-4o" +temperature = 0.5 +char_limit = 100000 + +[[4o.messages]] +role = "system" +content = "You are an extremely skilled programmer and scientist with a keen eye for detail and an emphasis on readable code. You have been tasked with acting as a smart version of the cat unix program. You take text and a prompt in and write text out. For that reason, it is of crucial importance to just write the desired output. Do not under any circumstance write any comment or thought as your output will be piped into other programs. Do not write the markdown delimiters for code as well. Sometimes you will be asked to implement or extend some input code. Same thing goes here, write only what was asked because what you write will be directly added to the user's editor. Never ever write ``` around the code. Make sure to keep the indentation and formatting. " + +[o1] +api = "openai" +model = "o1-preview" +char_limit = 100000 + +[[o1.messages]] +role = "user" +content = "You are an extremely skilled programmer and scientist with a keen eye for detail and an emphasis on readable code. You have been tasked with acting as a smart version of the cat unix program. You take text and a prompt in and write text out. For that reason, it is of crucial importance to just write the desired output. Do not under any circumstance write any comment or thought as your output will be piped into other programs. Do not write the markdown delimiters for code as well. Sometimes you will be asked to implement or extend some input code. Same thing goes here, write only what was asked because what you write will be directly added to the user's editor. Never ever write ``` around the code. Make sure to keep the indentation and formatting. " diff --git a/adm/grading/submission-qualitative.md b/adm/grading/submission-qualitative.md new file mode 100644 index 0000000..21e4134 --- /dev/null +++ b/adm/grading/submission-qualitative.md @@ -0,0 +1,16 @@ + - Course: DA2215 HT24 (vetcyb24-p2) + - Assignment: Reflection on qualitative methods + - Student: Studenten Studentensdotter + - Submission id: 24972065 + - Submitted at: 2024-12-11T14:54:21Z + - Graded at: None + - Grade: None + - Graded by: None + - Speedgrader: https://canvas.kth.se/... +# Body + +

Qualitative methods are very important for security research as they offer insights into the why and how of security-related behaviour, and decisions. They often involve human factors, e.g. why choose this password, how is the usability of this library. Quantitative data cannot give us this kind of information.

+

For example for statistical data (quantitative data), qualitative data can give insights into the why, it provides context and meaning.

+

 

+

 

+ diff --git a/adm/grading/test-grade_reflections-err-first.txt b/adm/grading/test-grade_reflections-err-first.txt new file mode 100644 index 0000000..6e67ee7 --- /dev/null +++ b/adm/grading/test-grade_reflections-err-first.txt @@ -0,0 +1,57 @@ +### sc +##### 1 +vetcyb24p2 user1@kth.se Reflection on the use of statistics: No + +The reflection lacks a direct focus on the use of statistics as it seems to divert more towards machine learning and cybersecurity without elaborating sufficiently on how statistics is specifically used and its implications. +##### 2 +vetcyb24p2 user1@kth.se Reflection on the use of statistics: No + +The reflection is vague and lacks a clear and relevant discussion about the use of statistics specifically. It briefly mentions statistics and machine learning but does not demonstrate a thorough reflection on the use of statistics as was asked for in the assignment. +##### 3 +##### 4 +vetcyb24p2 user1@kth.se Reflection on the use of statistics: No + +The reflection provided is quite general and lacks depth in discussing the specific use of statistics. The mention of machine learning and cybersecurity, while interesting, seems somewhat unrelated to the assignment's focus on reflecting on the use of statistics. +##### 5 +vetcyb24p2 user1@kth.se Reflection on the use of statistics: No, the reflection lacks a focused analysis on the specific use of statistics. The student briefly mentions statistics without providing a deeper or direct reflection on its application or implications, which was the central task. +### sc 4o +##### 1 +vetcyb24p2 user1@kth.se Reflection on the use of statistics: No + +The reflection does not specifically address the use of statistics in a detailed or relevant manner as requested. It briefly mentions statistics but quickly shifts focus to machine learning and cybersecurity without a thorough discussion on statistics. +##### 2 +vetcyb24p2 user1@kth.se Reflection on the use of statistics: No + +The reflection does not specifically address the use of statistics in a meaningful way. It briefly mentions statistics but lacks depth and relevance to the assignment prompt. The focus shifts to machine learning and cybersecurity, which are not directly related to the topic of statistics. +##### 3 +##### 4 +vetcyb24p2 user1@kth.se Reflection on the use of statistics: No + +The reflection does not specifically address the use of statistics in detail. It briefly mentions statistics as a tool but does not provide a relevant reflection on its use as required by the assignment. The mention of machine learning and cybersecurity is not directly related to the reflection on the use of statistics. +##### 5 +vetcyb24p2 user1@kth.se Reflection on the use of statistics: No + +The student's reflection lacks depth and relevance to the specific assignment on the use of statistics. The reflection is too general and does not specifically address the use of statistics in the context of the course or assignment. +### sc o1 +##### 1 +vetcyb24p2 user1@kth.se Reflection on qualitative methods: No + +The instruction is missing, so it's not possible to determine if the student made an honest effort. +##### 2 +vetcyb24p2 user1@kth.se Reflection on qualitative methods: No, the instruction was not provided, so cannot evaluate the student's reflection for relevance. +vetcyb24p2 user2@kth.se Reflection on qualitative methods: No + +The instruction was not provided, so it's not possible to determine if the student made an honest effort. +##### 3 +vetcyb24p2 user1@kth.se Reflection on the use of models, part I: No + +Cannot evaluate because the instruction is not provided. +vetcyb24p2 user2@kth.se Reflection on the use of models, part II: No. The instruction was not provided, so it's not possible to evaluate whether the student made an honest effort or attempted a relevant reflection based on what was asked. +vetcyb24p2 user3@kth.se Reflection on the use of models, part I: No + +The assignment instructions were not provided, so I cannot determine if the student has made an honest effort based on what was asked. +##### 4 +vetcyb24p2 user4@kth.se Reflection on the use of models, part I: No + +The instruction was not provided as context, so I cannot determine if the student made an honest effort. +##### 5 diff --git a/adm/grading/user1-statistics.md b/adm/grading/user1-statistics.md new file mode 100644 index 0000000..2cb8eea --- /dev/null +++ b/adm/grading/user1-statistics.md @@ -0,0 +1,18 @@ + - Course: DA2215 HT24 (vetcyb24-p2) + - Assignment: Reflection on the use of statistics + - Student: Student Studentensson + - Submission id: 24966368 + - Submitted at: 2024-12-11T19:42:23Z + - Graded at: None + - Grade: None + - Graded by: None + - Speedgrader: https://canvas.kth.se/courses/50613/gradebook/speed_grader?assignment_id=318595&student_id=154870 +## reflection.md + +```markdown +The statistic is an useful tool, that helps us capturing the big picture. But we need to take this into account as this can also mean that the individual cases may be quite different. + +The machine learning is an emerging field, that we need to understand as it changes everything and the cybersecurity is no exception. + +``` + diff --git a/adm/grading/user2-statistics.md b/adm/grading/user2-statistics.md new file mode 100644 index 0000000..5c5a654 --- /dev/null +++ b/adm/grading/user2-statistics.md @@ -0,0 +1,13 @@ + - Course: DA2215 HT24 (vetcyb24-p2) + - Assignment: Reflection on the use of statistics + - Student: Studenten Studentensson + - Submission id: 24966347 + - Submitted at: 2024-12-11T17:35:37Z + - Graded at: None + - Grade: None + - Graded by: None + - Speedgrader: https://canvas.kth.se/courses/50613/gradebook/speed_grader?assignment_id=318595&student_id=110079 +# Body + +

They used success rates from Mls trained with real-world and syntetic data to to compare how realistic data compares to syntetic. They also analyse false positive rates, precision and recall to measure effectivness in WF+WO attacks.  

+ diff --git a/adm/grading/user5-statistics.md b/adm/grading/user5-statistics.md new file mode 100644 index 0000000..b376c43 --- /dev/null +++ b/adm/grading/user5-statistics.md @@ -0,0 +1,14 @@ + - Course: DA2215 HT24 (vetcyb24-p2) + - Assignment: Reflection on the use of statistics + - Student: Stentina Studentensdotter + - Submission id: 24966371 + - Submitted at: 2024-12-10T15:05:24Z + - Graded at: None + - Grade: None + - Graded by: None + - Speedgrader: https://canvas.kth.se/courses/50613/gradebook/speed_grader?assignment_id=318595&student_id=173646 +# Body + +

Statistics play a vital role in security research for several reasons.
For example the first paper used live statics gathered by the Tor project to prove that their developed client closely matches a real network in performance. They essentially analysed patterns of the Tor networked and compared them with their developed simulation client.
The second paper, used statistics in their background section to quantify risk and the third paper measured the accuracy of WF classification an attacker can achieve in a real world scenario, which assessing the effectiveness of the attack.
There are of course many more (e.g. access risk based on history, evaluate performance, experiments to study user behaviour for something like password reuse), but I will stop at the ones found in the papers.

+

Machine learning essentially operates completely on the statistics of the data it was trained on. Therefore, all the things explained above can be done by an ML algorithm without a human having to spend time on it. However, an ML is only ever as good as the data it was trained on and also brings many other problems. Consequently, it is a useful tool but one need to be careful when using it and verify the results.

+ diff --git a/adm/grading/vetcyb-grading.sh.nw b/adm/grading/vetcyb-grading.sh.nw new file mode 100644 index 0000000..63e7681 --- /dev/null +++ b/adm/grading/vetcyb-grading.sh.nw @@ -0,0 +1,1169 @@ +\documentclass[a4paper]{article} +\usepackage[colorlinks]{hyperref} +\usepackage{noweb} +\noweboptions{shift,longchunks,longxref,breakcode} + +\usepackage[outputdir=ltxobj]{minted} + +%\usepackage[utf8]{inputenc} +%\DeclareUnicodeCharacter{25CF}{$\bullet$} +%\DeclareUnicodeCharacter{251C}{\mbox{\kern.23em +% \vrule height2.2exdepth1exwidth.4pt\vrule height2.2ptdepth-1.8ptwidth.23em}} +%\DeclareUnicodeCharacter{2500}{\mbox{\vrule height2.2ptdepth-1.8ptwidth.5em}} +%\DeclareUnicodeCharacter{2502}{\mbox{\kern.23em +% \vrule height2.2exdepth-1.8ptwidth.4pt\kern.23em}} +%\DeclareUnicodeCharacter{2514}{\mbox{\kern.23em +% \vrule height2.2exdepth-1.8ptwidth.4pt\vrule height2.2ptdepth-1.8ptwidth.23em}} + +\usepackage{cleveref} + +\author{Daniel Bosk} +\title{Reporting grades for vetcyb courses} + +\begin{document} +@ +\maketitle +\tableofcontents +\clearpage + +\section{Overview} + +We want to automate the reporting of the results in the vetcyb courses. +There are two ways to pass the course: +\begin{enumerate} + \item\label{ParticipationOption} By participating in the seminars and doing + the prep assignments. + Also known as, participating in all teaching. + \item\label{FinalProjectOption} By doing the final project. +\end{enumerate} + +We want a script that can be run by the [[grader]] script\footnote{% + See \texttt{https://github.com/dbosk/grader/}. +}. +This will have the standard design (see [[report_INL1]]), except that we'll do +the participation grading before. +This way we can set the grades of the INL1 assignments based on the +participation and prep. +<>= +#!/bin/bash + +courses="vetcyb(24-?p2|2[5-9]|[3-9][0-9])" +components="^INL1" +<> + +<> + +report_INL1() { + canvaslms results -c "${courses}" -A "${components}" \ + -S canvaslms.grades.conjunctavgsurvey \ + | sed -E "s/ ?[HV]T[0-9]{2}( \(.*\))?//" \ + | ladok report -fv +} + +grade_participation() { + <> +} + +main() { + grade_participation + report_INL1 +} + +# run main if not sourced from another script +if [ "$0" = "${BASH_SOURCE[0]}" ] +then + main +fi +@ + + +\section{Participation} + +That solved reporting the final project to LADOK (option +\ref{FinalProjectOption} above). +Now we need to translate the participation (option \ref{ParticipationOption}) +to grades for the INL1 assignments. +We want to grade the participation in the seminars and prep so that students +don't have to do the final project. + +The INL1 assignment group contains the following assignments: +\begin{verbatim} +INL1 Peer review: Designing a methodology to answer a question +INL1 Final version of the report for final seminar +INL1 The final seminar on 8-9/1 at 8:15 +INL1 Course evaluation survey +\end{verbatim} +We want to set grades on the first three assignments based on the participation +and prep. +We use a mandatory course evaluation to simply get a date. +If the student finishes by participation, they will finish a few week earlier. +CSN doesn't like this. +So to get a date in the last week of the course, we'll let the students who +finish by participation to do the mandatory course evaluation survey then. +That way we get a date in the last week of the course. +Also, we get all the students to fill in the survey, instead of the usual two +students. + +The participation contains the following assignments: +\begin{verbatim} +Participation INL1 Overview of Science in Security +Participation INL1 How to Design Computer Security Experiments +Participation INL1 How do you know it's secure? Passwords +Participation INL1 Reflection on Science in Security +Participation INL1 Comprehension, literature review: Of passwords and people +Participation INL1 Comprehension: Graphical Passwords: Learning from the First Twelve Years +Participation INL1 Achieving Rigor in Literature Reviews Insights from Qualitative Data Analysis and Tool-Support +Participation INL1 Live seminar 13/11 at 15:15 +Participation INL1 Comprehension: Of Passwords and People, Measuring the Effect of Password-Composition Policies +Participation INL1 Comprehension: Can long passwords be secure and usable? +Participation INL1 Comprehension: Why phishing works +Participation INL1 Live seminar 15/11 at 8:15 +Participation INL1 The RSA and ElGamal cryptosystems +Participation INL1 On the Security of EIGamal Based Encryption +Participation INL1 Stealing Keys from PCs using a Radio: Cheap Electromagnetic Attacks on Windowed Exponentiation +Participation INL1 Timing Analysis of Keystrokes and Timing Attacks on SSH +Participation INL1 Reflection on the use of models, part I +Participation INL1 Theorem proving: 1. Introduction +Participation INL1 Theorem proving: 2. Formal methods and Interactive Theorem Proving +Participation INL1 Theorem proving: 3. Example: Proving list-reverse correct (optional) +Participation INL1 Theorem proving: 4. Examples for what can be verified with Interactive Theorem Provers +Participation INL1 Theorem proving: 5. Limitations of Interactive Theorem Proving and Conclusion +Participation INL1 Model checking: algorithmic verification and debugging +Participation INL1 Reflection on the use of models, part II +Participation INL1 Live seminar 9/12 at 10:15 +Participation INL1 Dos and Don'ts of Machine Learning in Computer Security +Participation INL1 Reflection on the use of statistics +Participation INL1 Comprehension: Why Johnny can't encrypt +Participation INL1 Comprehension: Comparing the Usability of Cryptographic APIs +Participation INL1 Reflection on qualitative methods +Participation INL1 SoK: Science, Security and the Elusive Goal of Security as a Scientific Pursuit +Participation INL1 Live seminar 12/12 at 15:15 +\end{verbatim} + +This list will be useful. +However, it will be course specific. +We will generate it as follows. +We'll cache the results and only request it from Canvas if we don't have any +from before. +We'll refresh as often as the system's tempdir is cleaned. +<>= +assignments() { + local course="${1}" + <> + if [ ! -f "${assignments_cache}" ] + then + assignments_refresh "${course}" + fi + cat "${assignments_cache}" +} + +assignments_refresh() { + local course="${1}" + <> + canvaslms assignments -c "${course}" -A "${all_components}" \ + | cut -f 2-3 \ + > "${assignments_cache}" +} +<>= +all_components="INL1" +<>= +local assignments_cache="/tmp/${course}.assignments" +@ + +The idea is that we use the `Live seminar` assignments to tick off the `The +final seminar` assignment. +Then we let the written prep assignments tick off the `Peer review` assignment. +Let's write two functions that return the assignments for each of these. + +These functions will get the names of the assignments and filter on those +exactly. +However, if some are substrings of others, we might get false positives. +But this will not be any issue for us. +<>= +filter_participation() { + local course="${1}" + grep -f <(assignments "${course}" \ + | grep -E "^Participation INL1" \ + | cut -f 2) +} + +filter_live_seminar() { + local course="${1}" + filter_participation "${course}" \ + | grep -E 'Live seminar [0-9]{1,2}/[0-9]{1,2}' +} + +filter_peer_review() { + local course="${1}" + filter_participation "${course}" \ + | grep -vE 'Live seminar [0-9]{1,2}/[0-9]{1,2}' +} +@ + +We'll do the same to get the INL1 assignments. +<>= +filter_INL1() { + local course="${1}" + grep -f <(assignments "${course}" \ + | grep -E "^INL1" \ + | cut -f 2) +} +@ + +\subsection{Grading participation} + +As mentioned above, we'll need to do this on a per course basis, since the +assignments might vary from round to round. +<>= +oldIFS="${IFS}" +IFS=$'\n' +for course in $(<>) +do + IFS="${oldIFS}" + grade_course_participation "${course}" +done +IFS="${oldIFS}" +@ + +For the courses, we need something that can be used as an identifier. +That would be the first or second column. +The first is shorter, so we'll use that. +But it also contains parentheses, so we'll need to deal with them to not be +interpreted as regex. +If they're interpreted as regex, we won't get any matches. +So we'll need to either escape them or replace them by periods ([[.]]). +<>= +canvaslms courses "${courses}" \ +| cut -f 1 \ +| sed -E 's/([()])/\\\1/g' +@ + +Now to the grading. +We want to pass the INL1 assignments if the appropriate participation and prep +are checked. +However, we don't want to pass only one of the assignments, both must pass to +skip the project. +<>= +grade_course_participation() { + local course="${1}" + <> + <> + local passed=$(mktemp) + for student in ${students} + do + <> + done > "${passed}" + if [ -s "${passed}" ] + then + canvaslms grade -c "${course}" \ + -a "${project_assignments_regex}" \ + -g P \ + -u "$(make_regex ${passed})" \ + -m "${participation_msg}" + fi + <> +} +<>= +participation_msg=" +You did all prep and actively participated in all seminars. You don't have to +do the project. +" +@ + +We must remember to set the grades for only the project assignments, not all +INL1 assignments. +The INL1 assignments group also includes a survey. +<>= +project_assignments_regex="^(Peer review:|The final seminar|Final version of the report)" +@ + +The function [[make_regex]] simply takes the list of usernames [[user1]], +[[user2]] and [[user3]] (contained in the file supplied as the argument, one +per line) and turns them into the regex [[(user1|user2|user3)]]. +<>= +make_regex() { + echo -n "(" + paste -sd '|' "$@" \ + | sed -zE 's/\n//g' # remove newline introduced by paste + echo ")" +} +@ + +\subsection{Get [[results]] from Canvas} + +We need to get the results from Canvas. +We only do this if we don't already have them. +<>= +results() { + local course="${1}" + local results_cache="/tmp/${course}.results" + if [ ! -f "${results_cache}" ] + then + results_refresh "${course}" + fi + cat "${results_cache}" +} +@ + +When we get the results from Canvas, we want to filter out the columns we need. +We want the assignment name, the student's name, the grade and grading date. +The data will look like this: +\begin{verbatim} +Peer review: Designing a methodology to answer a question user@kth.se +The final seminar on 8-9/1 at 8:15 user@kth.se +Overview of Science in Security user@kth.se 100 2024-12-10T08:39:52Z +How to Design Computer Security Experiments user@kth.se 100 2024-12-10T08:40:39Z +How do you know it's secure? Passwords user@kth.se 100 2024-12-10T08:41:27Z +Reflection on Science in Security user@kth.se +Comprehension, literature review: Of passwords and people user@kth.se 100 2024-12-10T08:45:54Z +Comprehension: Graphical Passwords: Learning from the First Twelve Years user@kth.se 100 2024-12-10T08:47:16Z +Achieving Rigor in Literature Reviews Insights from Qualitative Data Analysis and Tool-Support user@kth.se 100 2024-12-10T08:47:41Z +Live seminar 13/11 at 15:15 user@kth.se complete 2024-11-18T13:22:02Z +Comprehension: Of Passwords and People, Measuring the Effect of Password-Composition Policies user@kth.se 100 2024-12-11T13:11:30Z +Comprehension: Can long passwords be secure and usable? user@kth.se 100 2024-12-11T13:12:19Z +Comprehension: Why phishing works user@kth.se 100 2024-12-11T13:13:05Z +Live seminar 15/11 at 8:15 user@kth.se complete 2024-11-18T12:57:48Z +The RSA and ElGamal cryptosystems user@kth.se 100 2024-12-11T13:14:08Z +On the Security of EIGamal Based Encryption user@kth.se 100 2024-12-11T13:14:52Z +Stealing Keys from PCs using a Radio: Cheap Electromagnetic Attacks on Windowed Exponentiation user@kth.se 100 2024-12-11T13:15:27Z +Timing Analysis of Keystrokes and Timing Attacks on SSH user@kth.se 100 2024-12-11T13:16:17Z +Reflection on the use of models, part I user@kth.se +Theorem proving: 1. Introduction user@kth.se 100 2024-12-11T13:20:07Z +Theorem proving: 2. Formal methods and Interactive Theorem Proving user@kth.se 90 2024-12-11T13:20:40Z +Theorem proving: 3. Example: Proving list-reverse correct (optional) user@kth.se 90 2024-12-11T13:21:04Z +Theorem proving: 4. Examples for what can be verified with Interactive Theorem Provers user@kth.se 100 2024-12-11T13:21:29Z +Theorem proving: 5. Limitations of Interactive Theorem Proving and Conclusion user@kth.se 90 2024-12-11T13:21:54Z +Model checking: algorithmic verification and debugging user@kth.se 90 2024-12-11T13:22:23Z +Reflection on the use of models, part II user@kth.se +Live seminar 9/12 at 10:15 user@kth.se complete 2024-12-11T13:25:20Z +Dos and Don'ts of Machine Learning in Computer Security user@kth.se +Reflection on the use of statistics user@kth.se +Comprehension: Why Johnny can't encrypt user@kth.se 100 2024-12-12T11:57:34Z +Comprehension: Comparing the Usability of Cryptographic APIs user@kth.se 100 2024-12-12T11:58:03Z +Reflection on qualitative methods user@kth.se +SoK: Science, Security and the Elusive Goal of Security as a Scientific Pursuit user@kth.se 100 2024-12-12T12:11:18Z +Live seminar 12/12 at 15:15 user@kth.se complete 2024-12-16T09:47:28Z +\end{verbatim} +When we fetch the results from Canvas, we want to use the [[-l]] option to get +their username instead of their name. +<>= +results_refresh() { + local course="${1}" + local results_cache="/tmp/${course}.results" + canvaslms submissions -c "${course}" -A "${all_components}" -l \ + | cut -f 2-4,6 \ + > "${results_cache}" +} +@ + +Here we must be able to distinguish assignments with different grades. +We already have [[filter_live_seminar]] to get some assignments with the grade +`complete`. +But we'll also need the reflection assignments and the non-reflection +assignments. +The reflections are graded `complete`, the non-reflections are graded at least +90 points. +<>= +filter_reflection() { + local course="${1}" + filter_participation "${course}" \ + | grep 'Reflection' +} + +filter_non_reflection() { + local course="${1}" + filter_participation "${course}" \ + | grep -v 'Reflection' +} +@ + + +\subsection{Finding which students to grade} + +The students that we need to grade are the ones that have not already passed +the assignments. +Fortunately, all the assignments in the INL1 group are pass/fail. +So we can just look for the students that have not passed the assignments. +<>= +students=$(results "${course}" | students_to_grade) +@ + +The idea is this. +We get the results from stdin. +That way we can use functions like [[filter_INL1]] (as above) to get the +assignments we are interested in. +We filter out the students in these results, +then we go through student-by-student to check if they passed all assignments +or not. +If they did not, we echo their username. +<>= +students_to_grade() { + local results="$(cat)" + <> + local students=$(echo "${results}" | cut -f 2 | sort -u) + for student in ${students} + do + <> + <> + echo "${student}" + done +} +@ + +We can define a regex for [[passing_grades]] that captures all passing grades +in all grading scales: +\begin{itemize} +\item P, +\item complete, but not incomplete, +\item 90--100. +\end{itemize} +<>= +local passing_grades="${1:-^(P|complete|9[0-9]|100)}" +@ + +If the student has passed any INL1 assignment, we don't need to grade them. +Then we know that they have either passed the course or started working on the +project. +<>= +if echo "${results}" | grep "${student}" \ + | filter_INL1 | passed_any "^P" +then + continue +fi +<>= +passed_any() { + local grades="${1}" + cut -f 3 | grep -qP "${grades}" +} +@ + +We don't need to grade anyone who is already graded, we don't want to spam them +with feedback. +So how do we detect this? +We can check if they have a grade on the `Final version of the report` +assignment, or rather, a date. +If that date is later than all the other dates, then they are graded. +Otherwise they're not. +<>= +local final_report_date=$(echo "${results}" \ + | grep "${student}" \ + | grep 'Final version of the report' \ + | cut -f 4) +local latest_date=$(echo "${results}" \ + | grep "${student}" \ + | filter_participation \ + | cut -f 4 \ + | sort \ + | tail -n 1) +if [ -n "${final_report_date}" -a -n "${latest_date}" ] +then + if [[ "${final_report_date}" > "${latest_date}" ]] + then + continue + fi +fi +@ We note that this only works when all assignments are in the results piped to +[[students_to_grade]]. +For instance, if we only get results filtered by [[filter_INL1]], then we we +can't compare the dates of the `Final version of the report` (which is present) +to the other assignments (which are not present). + +Also, when we give feedback to a student in the INL1 assignments, they are not +labelled as graded (with a grading date). +Perhaps we can get them to be labelled as graded if we set the grade [[-]]? +Nope, doesn't work. +We must set a non-passing grade on them. + +\subsection{Checking if a student has passed all assignments} + +The [[passed_all]] assignment will check if all the assignments that it gets +have a passing grade. +If so, it will return true; otherwise, false. + +It will take input from the standard input. +This will be on the same format as the [[results]] file. +That way we can get the grade column. +We'll count if there are as many P's as there are rows. + +We'll let an optional argument be the passing grade. +That will be a regex. +In some cases, we need negative lookbehind (requires [[-P]]) to avoid matching +`incomplete` when we're looking for `complete`. +On the other hand, since we compare the grade in the first column, we can make +use of [[^complete]] to avoid matching `incomplete`. +<>= +passed_all() { + <> + local lines=$(cat) + echo "${lines}" | cut -f 3 | grep -Pc "${passing_grades}" \ + | grep -qF "$(echo "${lines}" | wc -l)" +} +@ + + +\subsection{Check if a student has passed the participation} + +We'll simply check if they have the desired grades on all assignments. +<>= +passed_participation "${course}" "${student}" +<>= +passed_participation() { + local course="${1}" + shift + for student in $* + do + local student_results=$(results "${course}" | grep "${student}") + <> + then + continue + fi + <> + then + continue + fi + <> + then + continue + fi + echo "${student}" + done +} +@ + +We'll check if they have the grade `complete` on all live seminars. +If not, we continue to the next student. +<>= +if ! echo "${student_results}" \ + | filter_live_seminar "${course}" \ + | passed_all '^complete' +@ + +Otherwise we check if they also have at least 90 points on the FeedbackFruits +prep and `complete` on the reflections. +<>= +if ! echo "${student_results}" \ + | filter_peer_review "${course}" \ + | filter_non_reflection "${course}" \ + | passed_all '^9[0-9]|100' +<>= +if ! echo "${student_results}" \ + | filter_peer_review "${course}" \ + | filter_reflection "${course}" \ + | passed_all '^complete' +@ + +Now we want to construct a similar function, but this one should output +feedback on why the student didn't pass. +This function will take one student as argument and output the feedback on +stdout. +<>= +feedback_participation() { + local course="${1}" + local student="${2}" + local student_results=$(results "${course}" | grep "${student}" \ + | filter_participation) + if ! echo "${student_results}" | passed_all + then + <> + fi +} +@ This way we can use it to give feedback to the students who didn't pass. + +Now we should just find the assignments without a passing grade and print those +titles. +<>= +<> +then + echo "You didn't participate in the following live seminars:" + <> + echo "So you must do the project." +<> +then + echo "You haven't completed the following prep assignments" \ + echo "to a sufficient degree (90 points):" + <> + echo "But you can complete them to pass the course," \ + "without doing the project," \ + "since you participated in all seminars and completed most" \ + "of the prep assignments before the seminar." +fi +@ + +Let's look at those if statements. +We have +[[<>]] +from before. +We filter out the relevant assignments and check if they passed. +Then we filter out the ones without a passing grade and just cut the titles. + +Since the grade is in the second column, we prefix the grade by [[\s]] to avoid +matching undesired substrings. +<>= +echo "${student_results}" \ +| filter_live_seminar "${course}" \ +| grep -Ev '\scomplete' \ +| cut -f 1,3 +<>= +elif ! echo "${student_results}" \ + | filter_peer_review "${course}" \ + | passed_all +<>= +echo "${student_results}" \ +| filter_peer_review "${course}" \ +| filter_non_reflection "${course}" \ +| grep -Ev '\s9[0-9]|100' \ +| cut -f 1,3 +echo "${student_results}" \ +| filter_peer_review "${course}" \ +| filter_reflection "${course}" \ +| grep -Ev '\scomplete' \ +| cut -f 1,3 +@ + +\subsection{Giving feedback to students who didn't pass} + +The students who are listed in the file [[passed]] are the ones who passed. +The other students in [[students]] are the ones who didn't pass. +<>= +echo "${students}" | grep -vFxf "${passed}" +@ + +However, these results only make sense after the last seminar of the course. +Before that last seminar, none of the students could have passed. +Unfortunately, just checking if \emph{anyone} passed, will not be enough. +In the rounds following, maybe all missed a small thing, but then they won't +get feedback about it. +So we need to compare to the dates of the seminars. +We can't rely on the dates being equal, then the script might run before the +seminar that day. +So [[today]] must be strictly after the last seminar. +<>= +<> +<> +if [[ "${last_date}" < "${today}" ]] +then + for student in $(<>) + do + local feedback=$(feedback_participation "${course}" "${student}") + if [ -n "${feedback}" ] + then + canvaslms grade -c "${course}" -a "${project_assignments_regex}" \ + -u "${student}" \ + -m "${feedback}" + fi + done +fi +@ + + +\subsection{Getting the last seminar date} + +Now, the dates of the seminars are in the title of the assignments. +It's written as `Live seminar 13/11 at 15:15`. +So we want to change it into the date format `24-11-13`, that way it will be +comparable to today's date. +The relevant year is in the course name, so we can get it from +[[course]]---which will have the format [[vetcyb24p2]]. +<>= +last_seminar_date() { + local course="${1}" + local year=$(echo "${course}" | sed -E 's/vetcyb([0-9]{2})-?p[1-4]/\1/') + assignments "${course}" \ + | filter_live_seminar \ + | cut -f 2 \ + | cut -d ' ' -f 3 \ + | <> \ + | sort \ + | tail -n 1 +} +<>= +local year=$(echo "${course}" | sed -E 's/vetcyb([0-9]{2})-?p[1-4]/\1/') +local last_date=$(last_seminar_date "${course}") +@ + +We also need today's date in the same format. +<>= +local today=$(date +%y-%m-%d) +@ + +Let's rewrite the date to the format `yy-mm-dd`. +We must change the month and day to two-digit numbers. +We can do this by rewriting all [[/d]] and [[d/]] to [[/0d]] and [[0d/]]. +Then we can swap the month and day by rewriting [[dd/mm]] to [[mm-dd]] and +inserting the year ([[${year}]]) at the beginning. +<>= +sed -E "s|^([0-9]{1})/|0\1/|" \ +| sed -E "s|/([0-9]{1})$|/0\1|" \ +| sed -E "s|([0-9]{2})/([0-9]{2})|${year}-\2-\1|" +@ + + +\section{Grading the reflection assignments} + +We need to grade the reflection assignments. +We must also refresh the results from Canvas after we've graded these. + +To grade these assignments, we must download them from Canvas. +We can use [[canvaslms submission]] for this. +We must then check if the student has made an honest attempt at the assignment. +We can do this by asking GPT-4o to assess if it was an honest attempt, based on +the instruction. + +If we don't have any students to grade, we return 1 to indicate so. +This way we can skip refreshing all results if we didn't do anything here. +<>= +grade_reflections "${course}" ${students} \ + && results_refresh "${course}" +<>= +grade_reflections() { + local course="${1}" + shift + local students="$@" + <> + <> + <> + <> + if [ -z "${students_with_results}" ] + then + return 1 + fi + for student in ${students_with_results} + do + <> + done +} +@ + +To create the regex, we can use the [[make_regex]] function. +But that function requires a file containing all the students. +So we'll write the students who need grading to a temporary file and use that. + +We also emphasize \emph{students who need grading}. +Grading the reflections is, quite literally, an expensive operation. +But we only grade reflections that are not already graded---see +[[<>]] just +below. +<>= +local students_file=$(mktemp) +echo "${students}" | tr ' ' '\n' > "${students_file}" +students_regex=$(make_regex "${students_file}") +rm "${students_file}" +@ + +To download the submissions, we can use the [[canvaslms submission]] command. +We'll only download ungraded submissions (by the [[-U]] option to +[[canvaslms]]). +Consequently, we'll only grade submissions that haven't been graded yet. +<>= +local reflections_dir="/tmp/${course}.reflections.d" +<> +if [ ! -d "${reflections_dir}" ] +then + mkdir "${reflections_dir}" + canvaslms submission -c "${course}" \ + -a "${reflection_assign_regex}" \ + -u "${students_regex}" \ + -o "${reflections_dir}" \ + -U +fi +@ + +To construct the regex, we simply output all the reflection assignment titles +to a file and use [[make_regex]] on that file. +<>= +local reflection_assignments_file=$(mktemp) +assignments "${course}" | filter_reflection | cut -f 2 \ + > "${reflection_assignments_file}" +local reflection_assign_regex=$(make_regex "${reflection_assignments_file}") +@ + +Now we'll also download the assignment instructions. +We want to put them in the root of the [[reflections_dir]]. +<>= +local oldIFS="${IFS}" +IFS=$'\n' +for assignment in $(cat "${reflection_assignments_file}") +do + if [ ! -f "${reflections_dir}/${assignment}.md" ] + then + canvaslms assignment -c "${course}" -a "${assignment}" \ + > "${reflections_dir}/${assignment}.md" + fi +done +IFS="${oldIFS}" +@ + + +\subsection{Grading the student's reflections} + +The above gives us the following file structure +([[reflections_dir=reflections]], output from [[tree reflections]]): +\begin{verbatim} +reflections +├── user1@kth.se +│   └── DA2215 HT24 (vetcyb24-p2) +│   ├── Reflection on qualitative methods +│   │   └── metadata.md +│   ├── Reflection on Science in Security +│   │   └── metadata.md +│   ├── Reflection on the use of models, part I +│   │   └── metadata.md +│   ├── Reflection on the use of models, part II +│   │   ├── metadata.md +│   │   └── Reflection.txt +│   └── Reflection on the use of statistics +│   └── metadata.md +├── user2@kth.se +│   └── DA2215 HT24 (vetcyb24-p2) +│   ├── Reflection on ethics +│   │   └── metadata.md +│   ├── Reflection on Science in Security +│   │   └── metadata.md +│   ├── Reflection on the use of models, part I +│   │   └── metadata.md +│   ├── Reflection on the use of models, part II +│   │   ├── metadata.md +│   │   └── reflection_2.md +│   └── Reflection on the use of statistics +│   └── metadata.md +├── user3@kth.se +│   └── DA2215 HT24 (vetcyb24-p2) +│   ├── Reflection on qualitative methods +│   │   └── metadata.md +│   ├── Reflection on Science in Security +│   │   ├── da2215+Reflection+on+Science+in+Security.pdf +│   │   └── metadata.md +│   ├── Reflection on the use of models, part I +│   │   ├── metadata.md +│   │   └── Reflection+on+the+use+of+models%2C+part+I.pdf +│   ├── Reflection on the use of models, part II +│   │   ├── metadata.md +│   │   └── Reflection+on+the+use+of+models%2C+part+2.pdf +│   └── Reflection on the use of statistics +│   └── metadata.md +... +├── Reflection on ethics.md +├── Reflection on qualitative methods.md +├── Reflection on Science in Security.md +├── Reflection on the use of models, part III.md +├── Reflection on the use of models, part II.md +├── Reflection on the use of models, part I.md +├── Reflection on the use of statistics.md +... +213 directories, 227 files +\end{verbatim} +As we can see, some submissions (online text) are part of the [[metadata.md]] +file, other submissions appear as PDFs, markdown or plain text files. +However, [[canvaslms]] converts these files and integrates them into the +[[metadata.md]] file too (since version 4.5 it supports PDFs too). +So we only need the [[metadata.md]] file. +We also see that we have the instruction of each assignment in the root of the +directory. + +There will not be a directory for a student if they haven't submitted anything. +However, the reflection assignment instructions will also be here. +So we must filter out anything that doesn't contain an [[@]]. +<>= +local students_with_results=$(find "${reflections_dir}" \ + -mindepth 1 -maxdepth 1 -type d \ + | xargs -I {} basename "{}" \ + | grep "@") +@ + +The idea is that we use the [[llm]] package (and command) to ask GPT-4o to +assess if the submission is an honest attempt based on the instruction. +This means that for each student, we'll have to iterate through the +assignments. + +Since we used the [[-U]] flag to [[canvaslms submission]], each student's +directory will only contain assignment directories for the assignments they +have submitted that haven't been graded yet. +So we can iterate through those directories. +To find them, we want to look in the student's directory, just ignore the one +course directory (since it doesn't match the value of [[course]]), and pick the +assignment directories. +We'll use [[find]] for this. +Then we get a list of all the assignments for the student. +<>= +local student_assignments=$(find "${reflections_dir}/${student}" \ + -mindepth 2 -maxdepth 2 -type d) +local oldIFS="${IFS}" +IFS=$'\n' +for assignment_dir in ${student_assignments} +do + <> +done +IFS="${oldIFS}" +@ + +For each assignment, we'll need to check if the student has made an honest +attempt. +Their attempt is in the [[metadata.md]] file. +The instruction is in the root of the directory~[[reflections_dir]]. +<>= +local assignment=$(basename "${assignment_dir}") +local submission="$(cat ${assignment_dir}/metadata.md)" +if [ $? -ne 0 ] +then + echo "Couldn't find metadata.md for ${course} ${student} ${assignment}" >&2 + continue +elif [ -z "${submission}" ] +then + echo "No submission for ${course} ${student} ${assignment}" >&2 + continue +fi +local instruction="$(cat ${reflections_dir}/${assignment}.md)" +if [ -z "${instruction}" ] +then + echo "No instruction for ${course} ${assignment}" >&2 + continue +fi +local assessment=$(echo "${submission}" \ + | llm_cmd "${instruction}") +<>= +reflection_assessment_prompt=" +Based on the instruction (provided as context), evaluate if +the student has made an honest effort. That is, the student +attempted a relevant reflection based on what was asked for +in the instruction. If so, output 'Yes' followed by some +feedback on how to improve. If not, ouput 'No' and a brief +explanation. +" +<>= +llm_cmd() { + llm prompt \ + -m ${llm_model} \ + "${llm_options[@]}" \ + -s "${llm_sys_prompt}" \ + -f <(echo "${1}") \ + --no-stream \ + "${reflection_assessment_prompt}" +} +<>= +if echo "${assessment}" | grep -qi "^Yes" +then + echo "${course} ${student} ${assignment}: complete" + canvaslms grade -c "${course}" -a "${assignment}" -u "${student}" -g complete +else + echo >&2 + echo "${course} ${student} ${assignment}: NOT COMPLETE" >&2 + echo "${assessment}" >&2 + echo >&2 +fi +echo >&2 +@ + +An instruction might look like this: +\inputminted[breaklines]{markdown}{./instruction-qualitative.md} + +And a corresponding submission may look like this: +\inputminted[breaklines]{markdown}{./submission-qualitative.md} + + +\subsection{Evaluating the reliability} + +We want to evaluate the reliability of the grading. +We'll do this by running the [[grade_reflections]] functions on some select +students. + +In previous experiments (\cref{FirstExperiment}), some of the models give +unstable results. +The [[4o]] model with lowered temperature to \(0.5\) seems to give more +consistent results. +The [[o1]] (o1-preview at the time) model is much slower, but also seems to +give consistent results. +But now we'll run a few more systematic tests. +However, for some students, the [[o1]] model thinks that it didn't get a copy +of the instruction---which it did. +So we'll default to the [[4o]] class of models, using the latest (as of May +2025) version [[4.1]]. +<>= +llm_model="4.1" +llm_options=(-o temperature 0.5) +llm_sys_prompt=" +You are a skilled scientist and teacher with a keen eye +for detail and an emphasis on readability. + +You have been tasked with acting as a smart version of the +cat unix program. You take text and a prompt in and write +text out. For that reason, it is of crucial importance to +just write the desired output. Do not under any circumstance +write any comment or thought as your output will be piped +into other programs. +" +@ + +To do our evaluations, we used the following script. +It runs a few grading rounds on a few students for each model configuration. +Optimally, we'd run 20 rounds on all students, but that would take a long time +and be costly (but not more than a few euros). +<>= +#!/bin/bash + +source vetcyb-grading.sh + +course=vetcyb25p4 +#students=$(canvaslms users -sc ${course}) +students=" +user1@kth.se +user2@kth.se +user3@kth.se +user4@kth.se +user5@kth.se +" + +rounds=3 + +test() { + <> + + echo "### llm 4o temp=0.5" + echo "### llm 4o temp=0.5" >&2 + llm_model="4o" + llm_options=(-o temperature 0.5) + + <> + + echo "### llm 4.1" + echo "### llm 4.1" >&2 + llm_model="4.1" + unset llm_options + + <> + + echo "### llm 4.1 temp=0.5" + echo "### llm 4.1 temp=0.5" >&2 + llm_model="4.1" + llm_options=(-o temperature 0.5) + + <> + + echo "### llm o1" + echo "### llm o1" >&2 + llm_model="o1" + unset llm_options + + <> + + echo "### llm o3" + echo "### llm o3" >&2 + llm_model="o3" + unset llm_options + + <> + + echo "### llm o3-mini" + echo "### llm o3-mini" >&2 + llm_model="o3-mini" + unset llm_options + + <> + + echo "### llm o4-mini" + echo "### llm o4-mini" >&2 + llm_model="o4-mini" + unset llm_options + + <> +} +@ + +We need to force the download, in case we have already graded the students. +So this will be the same as above, except that we don't use the [[-U]] flag. +<>= +<> +local reflections_dir="/tmp/${course}.reflections.d" +<> +rm -Rf "${reflections_dir}" +canvaslms submission -c "${course}" \ + -a "${reflection_assign_regex}" \ + -u "${students_regex}" \ + -o "${reflections_dir}" +@ + +When running the tests, we want to output the headers to both stdout and +stderr. +<>= +for i in $(seq ${rounds}) +do + echo "##### $i" + echo "##### $i" >&2 + grade_reflections ${course} ${students} +done +@ + +When we run the tests, we want to filter stdout and stderr to separate files. +We're mostly interested in stderr, since that's where the fails appear. +And the fails are fewer than the passes. +<>= +test 1> >(tee test-grade_reflections.txt >&1) \ + 2> >(tee test-grade_reflections-err.txt >&1) \ + | tee test-grade_reflections-all.txt +@ + +Let's have a look at the output. +What's interesting to look at is the comments for when the model fails a +student, particularly how it compares across different rounds. +We want it to be stable, that is, to fail the same students in every round or +accept them in every round. +We want to avoid situations where the student might be accepted in one round +and rejected in another. +\inputminted[breaklines]{text}{test-grade_reflections-err.txt} + + + +\appendix + +\section{Results from the first experiment}\label{FirstExperiment} + +As we can see in the output, the [[4o]] model is stable. +Particularly, the [[o1]] model seems very unstable as it sometimes thinks that +it didn't get any copy of the instruction. +\inputminted[breaklines]{text}{test-grade_reflections-err-first.txt} + +Let's have a look at that reflection on statistics by user1. +We see that it is rejected four out of five times by both [[default]] and +[[4o]] configurations. +\inputminted[breaklines]{text}{./user1-statistics.md} + +Let's compare it to the instruction. +\inputminted[breaklines]{markdown}{./instruction-statistics.md} + +And now compare to some accepted reflections. +\inputminted[breaklines]{text}{./user5-statistics.md} +\inputminted[breaklines]{text}{./user2-statistics.md} + +The final run gave the following fails: +\begin{minted}{text} +$ grade_reflections vetcyb24p2 $(canvaslms users -sc vetcyb24p2 | cut -f 2) +vetcyb24p2 user1@kth.se Reflection on the use of statistics: No + +The reflection lacks depth and relevance specifically to the use of statistics. +It briefly mentions statistics but quickly shifts focus to machine learning and +cybersecurity without adequately reflecting on the use of statistics as +instructed. + +vetcyb24p2 user8@kth.se Reflection on Science in Security: No, the student +did not provide a reflection. The submission consists of a list of tasks and +activities related to security topics without any personal insights, analysis, +or reflection on the science in security. + +vetcyb24p2 user8@kth.se Reflection on qualitative methods: No. The student's +reflection focuses on quantitative aspects and usability testing rather than on +qualitative methods, which was the assignment's focus. + +vetcyb24p2 user9@kth.se Reflection on Science in Security: No. The submission +appears to be an analysis of specific security topics rather than a reflection +on Science in Security as requested by the assignment. +\end{minted} +This feedback was correct. +It was just the last one that was a bit brief in its description of methodology +for the different cases. + + +\section{The complete code} + +\inputminted[numbers=left]{bash}{vetcyb-grading.sh} + + +\end{document} diff --git a/course-design/objectives.tex b/course-design/objectives.tex deleted file mode 100644 index 2209d27..0000000 --- a/course-design/objectives.tex +++ /dev/null @@ -1,28 +0,0 @@ -\crefname{enumi}{}{} -\usepackage{xparse} -\NewDocumentCommand{\newLO}{m m m}{% - \expandafter\DeclareDocumentCommand{#1}{s}{% - \IfBooleanT{##1}{\label{#2}}% - #3% -}} - -\newLO{\LOrelate}{LOrelate} -{relate the different parts of scientific method, how they relate to one -another, contribute and not contribute to scientificity in security} - -\newLO{\LOevaluate}{LOevaluate} -{assess, analyse, and discuss the quality in, and ethical aspects of, knowledge -generation related to digital systems and in particular the security of these -systems} - -\newLO{\LOapply}{LOapply} -{apply scientific methodology to show how to answer issues in the cybersecurity -field} - -\newLO{\LOplan}{LOplan} -{plan and carry out assignments within given time frames and available -resources} - -\newLO{\LOcomm}{LOcomm} -{write short, clear and arguing texts based on own analysis as well as given -material} diff --git a/didactic b/didactic new file mode 160000 index 0000000..6545bfb --- /dev/null +++ b/didactic @@ -0,0 +1 @@ +Subproject commit 6545bfb9ba1e2fd4cbaf42709ec8fc1d99f4b53d diff --git a/makefiles b/makefiles index 6012491..898e836 160000 --- a/makefiles +++ b/makefiles @@ -1 +1 @@ -Subproject commit 601249138f77ccffa98a186b2fb966d1ea47e46d +Subproject commit 898e8365a5a2911841cb13be015b4e06382c5f5c diff --git a/course-design/.gitignore b/modules/00-intro/.gitignore similarity index 100% rename from course-design/.gitignore rename to modules/00-intro/.gitignore diff --git a/course-design/Makefile b/modules/00-intro/Makefile similarity index 90% rename from course-design/Makefile rename to modules/00-intro/Makefile index 6bd55ec..fd0e99a 100644 --- a/course-design/Makefile +++ b/modules/00-intro/Makefile @@ -19,5 +19,5 @@ clean: ${RM} notes.pdf slides.pdf -INCLUDE_MAKEFILES=../makefiles +INCLUDE_MAKEFILES=../../makefiles include ${INCLUDE_MAKEFILES}/tex.mk diff --git a/course-design/abstract.tex b/modules/00-intro/abstract.tex similarity index 100% rename from course-design/abstract.tex rename to modules/00-intro/abstract.tex diff --git a/course-design/contents.tex b/modules/00-intro/contents.tex similarity index 69% rename from course-design/contents.tex rename to modules/00-intro/contents.tex index faae245..bf6c3dc 100644 --- a/course-design/contents.tex +++ b/modules/00-intro/contents.tex @@ -1,5 +1,5 @@ \title{% - A Science of Security Course: An Overview + A Science of Security Course: Introduction } \author{Daniel Bosk\thanks{% This material was authored by Daniel Bosk and is available under the @@ -29,6 +29,66 @@ \section{Overview} +Let's have a look at provable security, widely considered the epitome of +security research. + +\begin{frame} + \begin{example}[\enquote{Provable security}]\label{SecurePassword} + A uniformly random string of length \(n\) is secure to use as a password. + We can prove it will take millions of years to guess it. + \end{example} + + \pause + + \begin{onlyenv} + \begin{remark} + \begin{itemize} + \item Attackers still get in, weird \dots + \end{itemize} + \end{remark} + \end{onlyenv} +\end{frame} + +\begin{frame} + \begin{example}[Usability] + \begin{itemize} + \item Turns out people can't handle uniformly random passwords. + \item Particularly not with a unique such password for every service. + \item They can't generate uniformly random passwords either. + \end{itemize} + \end{example} + + \pause + + \begin{remark}[Several aspects] + \begin{itemize} + \item We want students to handle complex problems. + \item Should see there are several aspects. + \item Aspects must be approached differently. + \end{itemize} + \end{remark} +\end{frame} + +However, attackers still get in. +% XXX add refs to password literature +It turns out that humans can't handle long, uniformly random passwords. +Particularly not when they have to remember one such password \emph{per service +they use}. +And they can't generate uniformly random passwords either! +None of these results could be found through provable security, they're from +the usability area of security and the methods are from the usability and +human--computer interaction research field. + +By this we don't say that provable security is wrong or doesn't work. +The same can be said of the usability methods. +Both are needed, they complement each other in generating understanding of +security problems. +The goal of the course is that the students should be able to master the +variety of methods and combine them to answer questions about security +problems. +That includes being able to evaluate different approaches to answering +questions and choosing among the most suitable ones. + \only{\subsection{The goal}} The goal of the course is to give a holistic view of the Science of Security. @@ -36,15 +96,15 @@ \section{Overview} \begin{frame} \begin{question} - What are the methods we (in the security community) use and why? + What are the \emph{research} methods we (in the security community) use and + why? \end{question} \end{frame} \begin{frame} \begin{block}{The goal} \begin{itemize} - \item Give a holistic view of Science of Security. - \item What are the methods we use and why? + \item Give a holistic view of \emph{Science} of Security. \end{itemize} \end{block} \end{frame} @@ -55,20 +115,23 @@ \section{Overview} \begin{frame}[fragile] \begin{example}\label{SoKProblem1} - \textcquote[\S IV]{SecurityAsAScience}{\textins*{C}laims of necessary - conditions for real-world security are unfalsifiable. - Claims of necessary conditions for formally-defined security are - tautological restatements of the assumptions}. + \textcquote[\S IV]{SecurityAsAScience}{% + \textins*{C}laims of necessary conditions for real-world security are + unfalsifiable. + Claims of necessary conditions for formally-defined security are + tautological restatements of the assumptions.% + } \end{example} \pause \begin{example}\label{SoKProblem2} \textcquote{SecurityAsAScience}{% - Unfalsifiable claims are common in security---and they, along with circular - arguments, are used to justify many defensive measures \textelp{} - \textins{T}here are many ways to argue measures in, but no way to argue one - out. + Unfalsifiable claims are common in security---and they, along with + circular arguments, are used to justify many defensive measures + \textelp{} + \textins{T}here are many ways to argue measures in, but no way to argue + one out.% } \end{example} \end{frame} @@ -84,12 +147,18 @@ \section{Overview} \begin{frame} \begin{block}{Aim of Science of Security course} \begin{itemize} - \item Complement the general methods course. + \item Complement the general methods course (TAMOS). \item Better prepare students for thesis (and hopefully worklife \dots). - \item They should be able to contribute to scientifically based + \item They should be able to contribute to \emph{scientifically} based development in cybersecurity. \end{itemize} \end{block} + + \begin{remark} + \begin{itemize} + \item This course is now fully synced with TAMOS\@. + \end{itemize} + \end{remark} \end{frame} \only
{\subsection{Goals and requirements}} @@ -103,7 +172,7 @@ \section{Overview} The requirements for Master's level are set out in \citetitle{HEO2}~\autocite{HEO2}, namely: -\begin{frame}[fragile] +%\begin{frame}[fragile] \begin{block}{Master's goals: Knowledge and understanding~\autocite{HEO2}} \only
{\begin{enumerate}[label={(K\arabic*)},ref=K\arabic*]} \only{\begin{itemize}} @@ -117,9 +186,9 @@ \section{Overview} \only
{\end{enumerate}} \only{\end{itemize}} \end{block} -\end{frame} +%\end{frame} -\begin{frame}[fragile] +%\begin{frame}[fragile] \begin{block}{Master's goals: Competence and skills~\autocite{HEO2}} \only
{\begin{enumerate}[label={(C\arabic*)},ref=C\arabic*]} \only{\begin{itemize}} @@ -139,9 +208,9 @@ \section{Overview} \only{\end{itemize}} \only
{\end{enumerate}} \end{block} -\end{frame} +%\end{frame} -\begin{frame}[fragile] +%\begin{frame}[fragile] \begin{block}{Master's goals: Judgement and approach~\autocite{HEO2}} \only
{\begin{enumerate}[label={(J\arabic*)},ref=J\arabic*]} \only{\begin{itemize}} @@ -154,7 +223,7 @@ \section{Overview} \only{\end{itemize}} \only
{\end{enumerate}} \end{block} -\end{frame} +%\end{frame} \paragraph{Intended learning outcomes}\label{LearningOutcomes} @@ -184,7 +253,19 @@ \section{Overview} in order to be able to contribute to scientifically based development. \end{frame} -\subsection{Prerequisites} +There are also two learning outcomes related more to general skill, rather than +scientific method: +\begin{frame}[fragile] + After passing the course, the student should be able to + \begin{itemize} + \item plan and carry out assignments within given time frames and using + available resources, and + \item write short, clear and arguing texts based on own analysis as well as + given material. + \end{itemize} +\end{frame} + +\subsection
{Prerequisites} The course requires some prerequisites. In the area of cybersecurity, the student should be able to: @@ -240,67 +321,6 @@ \section{More concretely} \subsection{The goal} -Let's have a look at provable security, widely considered the epitome of -security research. - -\begin{frame} - \begin{example}[\enquote{Provable security}] - \begin{itemize} - \item A uniformly random string of length \(n\) is the most secure - password. - - \item We can prove it will take millions of years to guess it. - \end{itemize} - \end{example} - - \pause - - \begin{remark} - \begin{itemize} - \item Attackers still get in, weird \dots - \end{itemize} - \end{remark} -\end{frame} - -\begin{example}[\enquote{Provable security}]\label{SecurePassword} - A uniformly random string of length \(n\) is the most secure password. - We can prove it will take millions of years to guess it. -\end{example} - -\begin{frame} - \begin{example}[Usability] - \begin{itemize} - \item Turns out people can't handle uniformly random passwords. - \item Particularly not with a unique such password for every service. - \item They can't generate uniformly random passwords either. - \end{itemize} - \end{example} - - \pause - - \begin{remark}[Several aspects] - \begin{itemize} - \item We want students to handle complex problems. - \item Should see there are several aspects. - \item Aspects must be approached differently. - \end{itemize} - \end{remark} -\end{frame} - -However, attackers still get in. -% XXX add refs to password literature -It turns out that humans can't handle long, uniformly random passwords. -Particularly not when they have to remember one such password \emph{per service -they use}. -And they can't generate uniformly random passwords either! -None of these results could be found through provable security, they're from -the usability area of security and the methods are from the usability and -human--computer interaction research field. - -By this we don't say that provable security is wrong or doesn't work. -The same can be said of the usability methods. -Both are needed, they complement each other in generating understanding of -security problems. The goal of the course is that the students should be able to master the variety of methods [\ref{LOrelate}] and combine them to answer questions about security problems [\ref{LOapply}]. @@ -312,93 +332,24 @@ \subsection{The goal} We can see this aspect in \ref{LOcomm} (and \ref{Ccomm}) and we will see this aspect as part of the assessment of the course later. - -\subsection{Teaching design} - -The teaching design of the course is particularly interesting to describe in -this course, because it is highly relevant on the meta level. - -\begin{frame}[fragile] - \begin{block}{Teaching design} +\begin{frame} + \begin{block}{Concrete goals} \begin{itemize} - \item Have problems that must be explored using several - methods. - \item Work through enough problems to cover the entire spectrum. + \item Ask good questions. + \item Propose suitable methods to answer them. + \item Evaluate the suitability of proposed methods. \end{itemize} \end{block} - \begin{remark}[Learning theory] - \begin{columns}[T] - \begin{column}{0.6\columnwidth} - \centering - \includegraphics[width=0.8\columnwidth]{figs/variatio-mater-studiorum.png} - \end{column} - \begin{column}{0.4\columnwidth} - \includegraphics[height=0.55\textheight]{figs/NCOL.jpg} - \end{column} - \end{columns} + \begin{remark} + \begin{itemize} + \item The focus is on \emph{research methods}. + \item It's \emph{not} on {constructing a secure solution}. + \item We want to know \emph{how do you know if it is secure?} + \end{itemize} \end{remark} \end{frame} -The design of the course is based on the variation theory of learning~\cite[see -\eg][]{NecessaryConditionsOfLearning}. -The goal of the theory is to achieve learning that prepares for unknown -situations. -It aims for deep learning (as opposed to surface learning, -\cf~\cite{DeepSurfaceLearning}). -This fits very well with our learning outcomes (\cref{LearningOutcomes}). -In fact, \textcite{NecessaryConditionsOfLearning} makes a point that learning -(according to this theory), is in fact very aligned with science and research. -However, variation theory, according to me, tells us more than the scientific -method traditionally does; it tells us about the preconditions to scientific -discovery. - -But what does this mean for the course concretely? -Well, in a sense, we could say that the teaching will treat some situations and -the assessment will treat something \enquote{completely different}. -Of course, there is something in common, namely the essence of the course. -But the purpose of the assessment is to test if the student can use what -they've learned from the teaching to handle this unknown situation---because -that's what the thesis and the rest of their careers are about. -Now this also means that if a student adopts the approach of opening the -assignments and then try to find the correct answers in the teaching material, -that student will be very disappointed in the course. - -In essence, we will look at problems and work out suitable methods to answer -questions about the problem at hand. -(This is the pattern of variation called contrast, in terms of -\cite{NecessaryConditionsOfLearning}, and focuses on \ref{LOrelate}, it's the -first and crucial step of learning.) -Then we'll dive into each method to explore what kinds of questions it can help -answer. -(This is the pattern of variation called generalization, in terms of -\cite{NecessaryConditionsOfLearning}, and focuses on \ref{LOevaluate}. -Coincidentally, this pattern corresponds to what is called the scientific -method.) -Finally, we will combine the methods to answer more complex questions. -(This is the variation of fusion, in terms of -\cite{NecessaryConditionsOfLearning}, and focuses on \ref{LOapply}.) -The assessment will then be a new problem where the students must combine the -methods to answer more complex questions on their own. - -For the remaining learning objectives, \ref{LOplan} and \ref{LOcomm}, we will -use the students' own works as learning material. -The idea is as follows: -The students will review each others' works, so this concerns \ref{LOcomm}. -If the work is about arguing why a method answers a particular research -question, then another student's solution will provide a contrasting -perspective to one's own. -Even if the choice of method is the same, the phrasing of the arguments -provides contrast to one's own arguments and phrasing thereof. - -We allow for \ref{LOplan} by giving the teaching material, assignments and -their deadlines for participating in seminars in advance. -Since the course runs in every period, it's up to the student to make a plan -for when to complete the assignments and participate in the seminars. -It's fine if a plan needs adapting, life happens; the important thing is that -the students learn to deal with it. -The assessment of this learning outcome consists of evaluating the plan, how it -turned out and what the student learned. \subsection{Format} @@ -418,22 +369,31 @@ \subsection{Format} The idea is that you learn from the first few seminars, so that we can grade you on the last one. -\begin{figure}[t] - \begin{fullwidth} - \subbottom[% - An interactive document in FeedbackFruits where one can answer quiz questions - posed in the document and ask questions about the content.% - \label{fbfdocquiz}% - ]{\includegraphics[height=0.3\textheight]{figs/fbf-doc-quiz-question.png}} - \hspace{1em} - \subbottom[% - An interactive document in FeedbackFruits where one can annotate the text - with focused topics together with other students in the class.% - \label{fbfdocannotation}% - ]{\includegraphics[height=0.3\textheight]{figs/fbf-doc-annotations.png}} - \caption{Two ways of having interactive documents.}\label{fbfdoc} - \end{fullwidth} -\end{figure} +\begin{figure*}[t] + \begin{sidecaption} + {Two ways of having interactive documents.} + [fbfdoc] + \begin{subfigure}[b]{0.49\columnwidth} + \centering + \includegraphics[width=\columnwidth]{figs/fbf-doc-annotations.png} + \caption{% + An interactive document in FeedbackFruits where one can annotate the + text with focused topics together with other students in the class.% + \label{fbfdocannotation}% + } + \end{subfigure} + \hfill + \begin{subfigure}[b]{0.49\columnwidth} + \centering + \includegraphics[width=\columnwidth]{figs/fbf-doc-quiz-question.png} + \caption{% + An interactive document in FeedbackFruits where one can answer quiz + questions posed in the document and ask questions about the content.% + \label{fbfdocquiz}% + } + \end{subfigure} + \end{sidecaption} +\end{figure*} \mode{% \begin{frame}[fragile] @@ -478,13 +438,9 @@ \subsection{Format} \begin{block}{Teaching material} \begin{itemize} \item<1> Video lectures where students can ask questions and answer - quizzes\footnote{FeedbackFruits or Canvas Studio}. - - \pause + quizzes. - \item<2-3> Reading assignments with social annotation\footnote{% - FeedbackFruits or Perusall - }. + \item<2-3> Reading assignments with social annotation. \end{itemize} \end{block} \end{column} @@ -531,18 +487,30 @@ \subsection{Format} \mode{% \begin{frame} - \begin{block}{Assessment} + \begin{block}{Teaching}<+> \begin{itemize} - \item A synchronous seminar to summarize all work and tie the sack. + \item Reading material and watching videos. + \item Write reflection to prepare for seminar. + \item Discuss and work with material at synchronous seminars. \end{itemize} \end{block} + + \begin{block}{Assessment}<+> + Either + \begin{enumerate} + \item You do all prep and participate in all seminars. + \item You do the project, write a report, peer review and participate in + the final seminar. + \end{enumerate} + \end{block} \end{frame} \begin{frame} \begin{block}{Giving the course} - \begin{enumerate} - \item Given every period; yes, four times per year. - \end{enumerate} + \begin{itemize} + \item Given twice pre year, for now: period 2 and 4. + \item If you don't finish, you can try again next round. + \end{itemize} \end{block} \end{frame} } @@ -611,7 +579,6 @@ \subsection{Contents} \begin{remark}[To focus on] \begin{itemize} \item What are the limitations? - \item Do these require a combination to form a Science of Security? \end{itemize} \end{remark} \end{frame} @@ -623,32 +590,29 @@ \subsection{Contents} \citetitle{SecurityAsAScience}\autocite{SecurityAsAScience} by \citeauthor{SecurityAsAScience}. -\mode{% -\begin{frame} - \begin{example}[Philosophy of Science of Security] - \begin{itemize} - \item Discuss - \citetitle{SecurityAsAScience}\footfullcite{SecurityAsAScience}. - \item What is Science of Security? - \item Does that even exist at the moment? - \item Shall we work according to the hypothetico-deductive model? - \item What are the problems? - \end{itemize} - \end{example} -\end{frame} -} + +\section{Contents} %\begin{frame} % \begin{block}{Contents, part II} % \begin{itemize} % \item General introductions to various subfields. % \item Which methods are used and why? -% \item Some exemplary papers? \alert<2>{Both good and bad!} +% %\item Some exemplary papers? \alert<2>{Both good and bad!} % \item How does a subfield fit into the holistic picture of Security? +% \item How can these methods fit into other situations? % \end{itemize} % \end{block} -%\end{frame} % +% \begin{exercise} +% \begin{itemize} +% \item What is an area or problem that you're interested in (in security)? +% \item Think of an interesting case that you can reflect on during the +% course. +% \end{itemize} +% \end{exercise} +%\end{frame} + %\begin{frame} % \begin{remark} % \begin{itemize} @@ -684,6 +648,137 @@ \subsection{Contents} % \end{block} %\end{frame} +\subsection{Learning} + +The first part of the course is an introduction to the scientific method in +general. +We're following the TAMOS course. +However, we complement that material with more security oriented examples. + +\paragraph{Scientificity, part 1} + +The first part is to read \citetitle{HowToDesignSecurityExperiments} by +\textcite{HowToDesignSecurityExperiments}. +This paper covers the same (albeit in different level of detail) as TAMOS' +lectures on +scientific inferences, +observation and measurement, +and experiments. +So after you've read the paper, you can help annotate the TAMOS videos with +more security-related examples. +Particularly, you should connect the pieces of +\citetitle{HowToDesignSecurityExperiments} and the TAMOS lectures. + +\begin{frame} + \begin{block}{Scientificity, part 1} + \begin{itemize} + \item \fullcite{HowToDesignSecurityExperiments} + \item Annotate TAMOS videos with security examples. + \begin{itemize} + \item Scientific inferences, + \item Observation and measurement, + \item Experiments. + \end{itemize} + \end{itemize} + \end{block} +\end{frame} + +\paragraph{Case study: Passwords} + +The next part is to look into passwords as a means of authentication. +Passwords as a mechanism is interesting because it can be evaluated in many +different ways---which covers the spectrum of methods discussed in TAMOS so +far, but also acts as a spring board to TAMOS' next topic (models). + +The first part of this topics will, however, be about scientific literature. +We'll go through some papers and look at how they deal with the literature. +(Still the subject focus is on passwords, but we're interested in the +methodological aspects.) +There will be a seminar given by the KTH Library. + +\begin{frame} + \begin{block}{Case study: Passwords} + \begin{itemize} + \item Different types of literature review. + \item Searching for literature. + \item Seminar by KTH Library. + \end{itemize} + \end{block} +\end{frame} + +\begin{frame} + \begin{block}{Case study: Passwords, continued} + \begin{itemize} + \item How do we evaluate the security of passwords? + \item Read and annotate a variety of papers exploring the topic. + \item Focus on the methodology. + \item Discuss during a seminar. + \end{itemize} + \end{block} +\end{frame} + +\begin{frame} + \begin{remark} + \begin{itemize} + \item I picked passwords because it's an easy topic to understand. + \item I also picked it because practice is still unscientifically + founded. + \item We're interested in the methodological aspects. + \item We want to connect what they do back to TAMOS. + \end{itemize} + \end{remark} +\end{frame} + +\paragraph{Models: Cryptography, formal methods, simulations} + +\begin{frame} + \begin{block}{Models---deductive methods} + \begin{itemize} + \item Cryptography: proving crypto protocols (RSA and ElGamal). + \item How models are inaccurate (side channels). + \item Formal methods: proving complex protocols, software, hardware. + \item Simulations: can't trial-and-error in real world (Tor). + \end{itemize} + \end{block} +\end{frame} + +\paragraph{Statistics} + +\begin{frame} + \begin{block}{Statistics} + \begin{itemize} + \item How is statistics used in security? + \item It's used in models and simulations (the science). + \item Used in security solutions (or attacks). + \end{itemize} + \end{block} +\end{frame} + +\paragraph{Qualitative methods} + +\begin{frame} + \begin{block}{Qualitative methods} + \begin{itemize} + \item How are qualitative methods used in security? + \item Qualitative methods are used to understand the human aspects of + security. + \item It's a method to explore things, to explain. + \item It can generate hypotheses that can be tested quantitatively. + \end{itemize} + \end{block} +\end{frame} + +\paragraph{Ethics} + +\begin{frame} + \begin{block}{Ethics} + \begin{itemize} + \item What should be do and not do? + \item What research questions should we answer and how can we do it? + \end{itemize} + \end{block} +\end{frame} + \subsection{Final assessment} The final assessment will be an assignment where the students design a research @@ -692,13 +787,52 @@ \subsection{Final assessment} LOrelate,LOcomm,LOplan,LOapply,LOevaluate% }. +\begin{frame} + \begin{block}{Final assessment} + \begin{itemize} + \item Design a research method for a given set of research questions. + \item Assess the suitability of the method. + \item Motivate the choices of methods. + \item Explain why the methods answer the questions. + \end{itemize} + \end{block} + + \begin{remark} + This will be done in groups. + \end{remark} +\end{frame} + You'll do this by writing a report motivating your choices of methods and explaining why they answer the question. You'll also present this report at the final seminar. Someone else will review the report, and you will review someone else's report too. -The final assessment will be done in pairs. +\begin{frame} + \begin{block}{Final seminar and review} + \begin{itemize} + \item Present this at the final seminar. + \item Review someone else's report. + \end{itemize} + \end{block} + + \begin{remark} + \begin{itemize} + \item Everyone should review one report (individually). + \item But it's fine to discuss in the groups. + \end{itemize} + \end{remark} +\end{frame} + +The final assessment will be done in groups of 3--5 students. + +\begin{frame} + \begin{remark} + \begin{itemize} + \item The final assessment will be done in groups of 3--5. + \end{itemize} + \end{remark} +\end{frame} %\begin{frame}[allowframebreaks] % \begin{block}{Assessment} @@ -720,13 +854,26 @@ \subsection{Final assessment} % \end{block} %\end{frame} -\begin{frame} +%\begin{frame} +% \begin{question} +% \begin{itemize} +% \item Comments, questions, other thoughts? +% \end{itemize} +% \end{question} +%\end{frame} + +\begin{onlyenv} +\section{About you} + +\begin{frame}[fragile] \begin{question} \begin{itemize} - \item Comments, questions, other thoughts? + \item What courses have you taken (or are taking)? + \item What are your interests? \end{itemize} \end{question} \end{frame} +\end{onlyenv} %%% REFERENCES %%% @@ -736,3 +883,94 @@ \subsection{Final assessment} \only{\printbibliography} \end{frame} + +\mode{\endinput} + +\subsection{Teaching design} + +The teaching design of the course is particularly interesting to describe in +this course, because it is highly relevant on the meta level. + +\begin{frame}[fragile] + \begin{block}{Teaching design} + \begin{itemize} + \item Have problems that must be explored using several + methods. + \item Work through enough problems to cover the entire spectrum. + \end{itemize} + \end{block} + + \begin{remark}[Learning theory] + \begin{columns}[T] + \begin{column}{0.6\columnwidth} + \centering + \includegraphics[width=0.8\columnwidth]{figs/variatio-mater-studiorum.png} + \end{column} + \begin{column}{0.4\columnwidth} + \includegraphics[height=0.55\textheight]{figs/NCOL.jpg} + \end{column} + \end{columns} + \end{remark} +\end{frame} + +The design of the course is based on the variation theory of learning~\cite[see +\eg][]{NecessaryConditionsOfLearning}. +The goal of the theory is to achieve learning that prepares for unknown +situations. +It aims for deep learning (as opposed to surface learning, +\cf~\cite{DeepSurfaceLearning}). +This fits very well with our learning outcomes (\cref{LearningOutcomes}). +In fact, \textcite{NecessaryConditionsOfLearning} makes a point that learning +(according to this theory), is in fact very aligned with science and research. +However, variation theory, according to me, tells us more than the scientific +method traditionally does; it tells us about the preconditions to scientific +discovery. + +But what does this mean for the course concretely? +Well, in a sense, we could say that the teaching will treat some situations and +the assessment will treat something \enquote{completely different}. +Of course, there is something in common, namely the essence of the course. +But the purpose of the assessment is to test if the student can use what +they've learned from the teaching to handle this unknown situation---because +that's what the thesis and the rest of their careers are about. +Now this also means that if a student adopts the approach of opening the +assignments and then try to find the correct answers in the teaching material, +that student will be very disappointed in the course. + +In essence, we will look at problems and work out suitable methods to answer +questions about the problem at hand. +(This is the pattern of variation called contrast, in terms of +\cite{NecessaryConditionsOfLearning}, and focuses on \ref{LOrelate}, it's the +first and crucial step of learning.) +Then we'll dive into each method to explore what kinds of questions it can help +answer. +(This is the pattern of variation called generalization, in terms of +\cite{NecessaryConditionsOfLearning}, and focuses on \ref{LOevaluate}. +Coincidentally, this pattern corresponds to what is called the scientific +method.) +Finally, we will combine the methods to answer more complex questions. +(This is the variation of fusion, in terms of +\cite{NecessaryConditionsOfLearning}, and focuses on \ref{LOapply}.) +The assessment will then be a new problem where the students must combine the +methods to answer more complex questions on their own. + +For the remaining learning objectives, \ref{LOplan} and \ref{LOcomm}, we will +use the students' own works as learning material. +The idea is as follows: +The students will review each others' works, so this concerns \ref{LOcomm}. +If the work is about arguing why a method answers a particular research +question, then another student's solution will provide a contrasting +perspective to one's own. +Even if the choice of method is the same, the phrasing of the arguments +provides contrast to one's own arguments and phrasing thereof. + +We allow for \ref{LOplan} by giving the teaching material, assignments and +their deadlines for participating in seminars in advance. +Since the course runs in every period, it's up to the student to make a plan +for when to complete the assignments and participate in the seminars. +It's fine if a plan needs adapting, life happens; the important thing is that +the students learn to deal with it. +The assessment of this learning outcome consists of evaluating the plan, how it +turned out and what the student learned. + + diff --git a/course-design/figs/GHurl.png b/modules/00-intro/figs/GHurl.png similarity index 100% rename from course-design/figs/GHurl.png rename to modules/00-intro/figs/GHurl.png diff --git a/course-design/figs/NCOL.jpg b/modules/00-intro/figs/NCOL.jpg similarity index 100% rename from course-design/figs/NCOL.jpg rename to modules/00-intro/figs/NCOL.jpg diff --git a/course-design/figs/fbf-doc-annotate-instruction.png b/modules/00-intro/figs/fbf-doc-annotate-instruction.png similarity index 100% rename from course-design/figs/fbf-doc-annotate-instruction.png rename to modules/00-intro/figs/fbf-doc-annotate-instruction.png diff --git a/course-design/figs/fbf-doc-annotations.png b/modules/00-intro/figs/fbf-doc-annotations.png similarity index 100% rename from course-design/figs/fbf-doc-annotations.png rename to modules/00-intro/figs/fbf-doc-annotations.png diff --git a/course-design/figs/fbf-doc-interactive-instruction.png b/modules/00-intro/figs/fbf-doc-interactive-instruction.png similarity index 100% rename from course-design/figs/fbf-doc-interactive-instruction.png rename to modules/00-intro/figs/fbf-doc-interactive-instruction.png diff --git a/course-design/figs/fbf-doc-quiz-question.png b/modules/00-intro/figs/fbf-doc-quiz-question.png similarity index 100% rename from course-design/figs/fbf-doc-quiz-question.png rename to modules/00-intro/figs/fbf-doc-quiz-question.png diff --git a/course-design/figs/fbf-video-contrib.png b/modules/00-intro/figs/fbf-video-contrib.png similarity index 100% rename from course-design/figs/fbf-video-contrib.png rename to modules/00-intro/figs/fbf-video-contrib.png diff --git a/course-design/figs/fbf-video-question.png b/modules/00-intro/figs/fbf-video-question.png similarity index 100% rename from course-design/figs/fbf-video-question.png rename to modules/00-intro/figs/fbf-video-question.png diff --git a/course-design/figs/fbf-video-quiz.png b/modules/00-intro/figs/fbf-video-quiz.png similarity index 100% rename from course-design/figs/fbf-video-quiz.png rename to modules/00-intro/figs/fbf-video-quiz.png diff --git a/course-design/figs/fbf-video-req-quiz.png b/modules/00-intro/figs/fbf-video-req-quiz.png similarity index 100% rename from course-design/figs/fbf-video-req-quiz.png rename to modules/00-intro/figs/fbf-video-req-quiz.png diff --git a/course-design/figs/variatio-mater-studiorum.png b/modules/00-intro/figs/variatio-mater-studiorum.png similarity index 100% rename from course-design/figs/variatio-mater-studiorum.png rename to modules/00-intro/figs/variatio-mater-studiorum.png diff --git a/modules/00-intro/notes.tex b/modules/00-intro/notes.tex new file mode 100644 index 0000000..87128ea --- /dev/null +++ b/modules/00-intro/notes.tex @@ -0,0 +1,21 @@ +\documentclass[a4paper,10pt,article,oneside]{memoir} +\let\subsubsection\subsection +\let\subsection\section +\let\section\chapter +\marginparmargin{outer} +\usepackage{subcaption} +%\newsubfloat{figure}% Allow subfloats in figure environment + +\input{preamble.tex} + +\usepackage[noamsthm,notheorems]{beamerarticle} +\setjobnamebeamerversion{slides} + +\usepackage[inline]{enumitem} + +\usepackage{authblk} +\let\institute\affil + +\begin{document} +\input{contents.tex} +\end{document} diff --git a/modules/00-intro/objectives.tex b/modules/00-intro/objectives.tex new file mode 100644 index 0000000..495c430 --- /dev/null +++ b/modules/00-intro/objectives.tex @@ -0,0 +1,28 @@ +\crefname{enumi}{}{} +\usepackage{xparse} +\NewDocumentCommand{\newLO}{m m m}{% + \expandafter\DeclareDocumentCommand{#1}{s}{% + \IfBooleanT{##1}{\label{#2}}% + #3% +}} + +\newLO{\LOrelate}{LOrelate} +{\emph{relate} the different parts of scientific method, how they relate to one +another, contribute and not contribute to scientificity in security} + +\newLO{\LOevaluate}{LOevaluate} +{\emph{assess, analyse, and discuss} the quality in, and ethical aspects of, +knowledge generation related to digital systems and in particular the security +of these systems} + +\newLO{\LOapply}{LOapply} +{\emph{apply} scientific methodology to show how to answer issues in the +cybersecurity field} + +\newLO{\LOplan}{LOplan} +{\emph{plan and carry out} assignments within given time frames and available +resources} + +\newLO{\LOcomm}{LOcomm} +{\emph{write short, clear and arguing} texts based on own analysis as well as +given material} diff --git a/course-design/preamble.tex b/modules/00-intro/preamble.tex similarity index 95% rename from course-design/preamble.tex rename to modules/00-intro/preamble.tex index 091f6e0..c4ab745 100644 --- a/course-design/preamble.tex +++ b/modules/00-intro/preamble.tex @@ -2,6 +2,7 @@ \usepackage[T1]{fontenc} \usepackage[british]{babel} \usepackage{booktabs} +\usepackage{subcaption} \usepackage[% natbib, @@ -50,5 +51,7 @@ \usepackage{multicol} \usepackage{changepage} +\usepackage{didactic} + \usepackage[capitalize]{cleveref} \input{objectives.tex} diff --git a/modules/00-intro/slides.tex b/modules/00-intro/slides.tex new file mode 100644 index 0000000..06fcc1b --- /dev/null +++ b/modules/00-intro/slides.tex @@ -0,0 +1,8 @@ +\documentclass[ignoreframetext]{beamer} +\input{preamble.tex} + +\begin{document} +\mode +\input{contents.tex} +\mode* +\end{document} diff --git a/modules/02-passwd/.gitignore b/modules/02-passwd/.gitignore index 85d632d..701f4cf 100644 --- a/modules/02-passwd/.gitignore +++ b/modules/02-passwd/.gitignore @@ -2,3 +2,4 @@ notes.pdf slides.pdf objectives.tex literature +seminar-slides.pdf diff --git a/modules/02-passwd/01-seminar.md b/modules/02-passwd/01-seminar.md new file mode 100644 index 0000000..756cd4f --- /dev/null +++ b/modules/02-passwd/01-seminar.md @@ -0,0 +1,14 @@ +The library will come and talk about tools to use for searching and how to do +literature reviews. + +**Participation**: Online in the class Zoom room. Working webcam and microphone +is required. + +**Preparation**: The assignments that you just did (all previous material). You +need a device that you can work with web tools on. + +**Seminar content**: The library will talk about different types of literature +reviews. They will also present some useful tools for searching for literature. +We'll use these tools to search for literature to answer some of the research +questions we've discussed in the preparation material. (That's why you need a +device to work on.) diff --git a/modules/02-passwd/02-seminar.md b/modules/02-passwd/02-seminar.md new file mode 100644 index 0000000..f5f43bd --- /dev/null +++ b/modules/02-passwd/02-seminar.md @@ -0,0 +1,12 @@ +**Participation**: Online in the class Zoom room. Working webcam and microphone +is required. + +**Preparation**: The assignments that you just did (all previous material from +last seminar until now). + +**Seminar content**: We will discuss the papers that you've worked with. + +Particularly, we'll discuss how they relate to the research questions that was +the origin of the discussion. What we're interested in is how well these papers +answer those research questions and, particularly, how we can answer *exactly* +those questions. diff --git a/modules/02-passwd/2024-04-05-Note-14-56.xoj b/modules/02-passwd/2024-04-05-Note-14-56.xoj new file mode 100644 index 0000000..985fee8 Binary files /dev/null and b/modules/02-passwd/2024-04-05-Note-14-56.xoj differ diff --git a/modules/02-passwd/Makefile b/modules/02-passwd/Makefile index 62fe324..8f47b51 100644 --- a/modules/02-passwd/Makefile +++ b/modules/02-passwd/Makefile @@ -1,5 +1,5 @@ .PHONY: all -all: notes.pdf slides.pdf +all: notes.pdf slides.pdf seminar-slides.pdf LATEXFLAGS+= -shell-escape @@ -16,7 +16,9 @@ notes.pdf: ${SRC} ${DEPENDS} ${FIGS} slides.pdf: slides.tex slides.pdf: ${SRC} ${DEPENDS} ${FIGS} -objectives.tex: ../../course-design/objectives.tex +seminar-slides.pdf: seminar-slides.tex seminar.tex + +objectives.tex: ../00-intro/objectives.tex ${DEPENDS}: ${LN} $< $@ diff --git a/modules/02-passwd/contents.tex b/modules/02-passwd/contents.tex index 4aba4a6..0e2bcc3 100644 --- a/modules/02-passwd/contents.tex +++ b/modules/02-passwd/contents.tex @@ -24,23 +24,19 @@ \clearpage -\section{Introduction} +\section
{Introduction} -\begin{frame} We've had passwords for about as long as we've had computers. Unfortunately, we still\footnote{As of \today.} haven't managed to do it right in practice. -\end{frame} So here we'll deal with the following question. -\begin{frame} - \begin{question}\label{RQ} - How can we know how secure our password-based authentication system will be? - \end{question} -\end{frame} +\begin{question}\label{RQ} + How can we know how secure our password-based authentication system will be? +\end{question} -\section{How do we know it's secure?} +\section[How do we know?]{How do we know it's secure?} \begin{frame} We have a system where users log in. @@ -48,16 +44,17 @@ \section{How do we know it's secure?} We've decided to use a password-based authentication system\footnote{% Yes, I know it's a bit of an oxymoron, but humor me. }. -\end{frame} -\begin{frame} \begin{exercise} - What do we need to know to try to answer \cref{RQ}? + What do we need to know to try to answer + \only
{\cref{RQ}}% + \only{how secure this will be}% + ? \end{exercise} \end{frame} -\section{What do we mean by secure?} +\section[Define secure?]{What do we mean by secure?} Well, first of all, we need to define what we mean by \enquote{being secure}. \Cref{RQ} asks us to estimate how secure a password-based authentication system @@ -123,23 +120,24 @@ \subsection{Literature reviews} \begin{frame} \begin{exercise} - We want to investigate the most common definitions of security for - authentication systems and for which types of authentication systems + We want to investigate \alert<2>{the most common} definitions of security + for authentication systems and for which types of authentication systems they're used. Which type of literature review should we do; systematic or non-systematic? Why? \end{exercise} -\end{frame} + \begin{onlyenv}
In this case, the goal is to get an overview of the literature. Since purpose is to research what definitions there are and how they're used, we must document our method of research. This means that we should do a systematic literature review. + \end{onlyenv} -\begin{frame} \begin{exercise} - We want to explore different definitions of security for authentication - systems to find a definition to use for our study. + We want to \alert<2>{explore \only<2>{\textins{some} }different} + definitions of security for authentication systems to find a definition to + use for our study. Which type of literature review should we do; systematic or non-systematic? Why? \end{exercise} @@ -279,14 +277,10 @@ \subsection{Deductive evaluation} \end{frame} A uniform distribution means that -\begin{frame} - all passwords are equally likely (\(\frac{1}{N^n}\)) -\end{frame} +all passwords are equally likely (\(\frac{1}{N^n}\)) and that -\begin{frame} - the Shannon entropy is maximized and equal to \(-\log \frac{1}{N^n} = n \log - {N}\), -\end{frame} +the Shannon entropy is maximized and equal to \(-\log \frac{1}{N^n} = n \log +{N}\), where \(N\) is the number of possible characters and \(n\) is the length of the password. @@ -306,6 +300,14 @@ \subsection{Deductive evaluation} Is it really secure, why or why not? How can we answer this question? \end{exercise} + + \begin{solution} + We can try forcing some user-generated passwords. + + We've assumed that the passwords are uniformly distributed. + But are they? + How can we find out? + \end{solution} \end{frame} \subsection{Empirical evaluation} @@ -328,10 +330,10 @@ \subsection{Empirical evaluation} So we should change the question into the following. \begin{frame} -\begin{question} +\begin{question}[Password distribution] How does different password policies affect the password distribution? \end{question} -\begin{question}\label{Guessability} +\begin{question}[Password distribution, guessability]\label{Guessability} How easily can we guess the passwords under different password policies? \end{question} \begin{exercise} @@ -349,6 +351,16 @@ \subsection{Empirical evaluation} We will explore these papers to see how they tried to answer these questions, so we'll return to them. +\begin{frame} + \begin{example}[Password distribution, guessability] + \fullcite{OfPasswordsAndPeople} + \end{example} + + \begin{example}[Guessability, usability] + \fullcite{CanLongPasswordsBeSecureAndUsable} + \end{example} +\end{frame} + However, we can do other estimates deductively too. For instance, we have the very famous \enquote{correct horse battery staple} from xkcd (\cref{xkcd936}). @@ -358,15 +370,22 @@ \subsection{Empirical evaluation} \begin{frame} \begin{figure}[h] - \includegraphics[width=\linewidth]{fig/password_strength.png} - \caption{% - The famous xkcd \enquote{correct horse battery staple} comic. + \begin{sidecaption}[xkcd 936]{% + Illustrating the password policy of using a long passphrase. Image: xkcd.com/936/. - }\label{xkcd936} + }[xkcd936] + \includegraphics[width=\linewidth]{fig/password_strength.png} + \end{sidecaption} \end{figure} \end{frame} -\section{But is it even a good model to begin with?} +\begin{frame} + \begin{exercise} + Did any of the papers answer the question of how the passwords are chosen? + \end{exercise} +\end{frame} + +\section[Is it a good model?]{But is it even a good model to begin with?} \begin{frame} \begin{exercise} @@ -385,6 +404,7 @@ \section{But is it even a good model to begin with?} \begin{frame} \begin{question} Can the verifier be an adversary or is the verifier always benign? + What are the consequences of this? \end{question} \end{frame} @@ -395,30 +415,74 @@ \section{But is it even a good model to begin with?} whether the verifier is benign or not by entering their password: the reasoning was that if the is correct it will accept the password, otherwise it will not (since it doesn't know the password)---which is a fallacy. +This tells us that we need to consider the verifier as an adversary, +consequently we need a zero-knowledge property in \cref{FormalSecurity}. + +\begin{frame} + \begin{example} + \fullcite{WhyPhishingWorks} + \end{example} + + \pause + + \begin{example}[Consequences] + We need the zero-knowledge property in our security definition. + (We actually need \emph{malicious}, not honest, verifier zero-knowledge.) + \end{example} +\end{frame} We can also again turn to the wisdom of xkcd for another argument against the benign verifier assumption, namely password reuse (\cref{xkcd792}). \begin{frame} \begin{figure} - \includegraphics[height=0.9\textheight]{fig/password_reuse.png} - \caption{% + \begin{sidecaption}[xkcd 792]{% Illustrating whether the benign verifier assumption is a good idea in practice. Image: xkcd.com/792/. - }\label{xkcd792} + }[xkcd792] + \flushscap + \includegraphics[height=0.9\textheight]{fig/password_reuse.png} + \end{sidecaption} \end{figure} \end{frame} -\paragraph{Case study} - -We will study the paper by \textcite{WhyPhishingWorks} to explore how they -answered this question. -So we'll return to this paper later. - %\begin{frame} % \begin{question} % Are there more reasons? % Password re-use, incompetent service, malicious service. % \end{question} %\end{frame} + +\paragraph{Case study} + +We will study the paper by \textcite{WhyPhishingWorks} to explore how they +answered this question. +So we'll return to this paper later. + + +\section{Conclusion} + +\begin{frame} + We might need a qualitative (\eg usability) study + \only{\newline}% + to inform our deductive (\eg cryptographic) choices. +\end{frame} + +\begin{frame} + \begin{question}[Is the opposite also true?] + Can deduction be a useful (necessary?) complement for empirical studies? + How? + \end{question} +\end{frame} + +For instance, using a deductive approach we can rule out all those approaches +to passwords that are \emph{provably} easy to guess. +There is no need to empirically test them, which takes time, if we already know +they're bad. + +\begin{frame} + \begin{exercise} + What other aspects didn't we consider? + \end{exercise} +\end{frame} diff --git a/modules/02-passwd/grade.sh b/modules/02-passwd/grade.sh new file mode 100644 index 0000000..de51b62 --- /dev/null +++ b/modules/02-passwd/grade.sh @@ -0,0 +1,45 @@ +oldIFS=$IFS +IFS=$'\n' + +course="vetcyb24p2" +seminar_date=${1} +file=${2} + +teachers="(dbosk|masvanbe|elzen)@kth.se" + +get_grade_data() { + tail -n +2 ${file} \ + | grep -Ev "${teachers}" \ + | cut -d, -f 2,3,5 \ + > participation-seminar.csv + + for user in $(cat participation-seminar.csv | cut -d, -f 1 | sort -u) + do + echo -n "$user "; + sum_minutes=0; + for minutes in $(grep $user participation-seminar.csv | cut -d, -f 3) + do + sum_minutes=$(($sum_minutes + $minutes)); + done; + echo $sum_minutes; + done +} + +open ${file} & +get_grade_data +read -p "Enter the limit in minutes: " limit_minutes + +for result in $(get_grade_data) +do + user=$(echo $result | cut -d ' ' -f 1); + minutes=$(echo $result | cut -d ' ' -f 2); + if [ $minutes -ge $limit_minutes ] + then + canvaslms grade -c ${course} -a "Live seminar ${seminar_date}" \ + -u $user -g complete + else + canvaslms grade -c ${course} -a "Live seminar ${seminar_date}" \ + -u $user -g incomplete \ + -m "You had $minutes minutes of participation out of more than $limit_minutes minutes." + fi +done diff --git a/modules/02-passwd/notes.tex b/modules/02-passwd/notes.tex index 49b372d..20e181f 100644 --- a/modules/02-passwd/notes.tex +++ b/modules/02-passwd/notes.tex @@ -1,39 +1,4 @@ \documentclass[a4paper,10pt,article,oneside]{memoir} -%%% Tufte %%% -\usepackage{marginfix} -%\setlength{\evensidemargin}{\oddsidemargin} -\marginparmargin{outer} -\setlrmarginsandblock{2.5cm}{8cm}{*} - -\footnotesinmargin - -\usepackage{ragged2e} -\renewcommand{\sidefootform}{\RaggedRight} -\renewcommand{\foottextfont}{\footnotesize\RaggedRight} - -\setmpjustification{\RaggedRight}{\RaggedRight} - -% margin figure and caption typeset ragged against text block -\setfloatadjustment{marginfigure}{\mpjustification} -\setmarginfloatcaptionadjustment{figure}{\captionstyle{\mpjustification}} - -% From https://tex.stackexchange.com/a/324757/17418 -% Palatino for main text and math -\usepackage[osf,sc]{mathpazo} - -% Helvetica for sans serif -% (scaled to match size of Palatino) -\usepackage[scaled=0.90]{helvet} - -% Bera Mono for monospaced -% (scaled to match size of Palatino) -\usepackage[scaled=0.85]{beramono} - -\setlxvchars\setxlvchars -\checkandfixthelayout - -\nouppercaseheads -%%% end tufte %%% \let\subsubsection\subsection \let\subsection\section \let\section\chapter @@ -46,17 +11,6 @@ %\usepackage{authblk} %\let\institute\affil -\declaretheorem[style=theorem]{theorem} -\declaretheorem[style=definition]{definition} -\declaretheorem[style=definition]{assumption} -\declaretheorem[style=definition]{protocol} -\declaretheorem[style=example]{example} -\declaretheorem[style=remark]{remark} -\declaretheorem[style=remark]{idea} -\declaretheorem[style=exercise]{exercise} -\declaretheorem[style=exercise]{question} -\declaretheorem[style=solution]{solution} - \begin{document} \input{contents.tex} diff --git a/modules/02-passwd/preamble.tex b/modules/02-passwd/preamble.tex index dce955a..43bbd69 100644 --- a/modules/02-passwd/preamble.tex +++ b/modules/02-passwd/preamble.tex @@ -50,6 +50,7 @@ \usepackage{multicol} \usepackage{changepage} +\usepackage{didactic} \usepackage[capitalize]{cleveref} \input{objectives.tex} diff --git a/modules/02-passwd/seminar-slides.tex b/modules/02-passwd/seminar-slides.tex new file mode 100644 index 0000000..3131b46 --- /dev/null +++ b/modules/02-passwd/seminar-slides.tex @@ -0,0 +1,12 @@ +\documentclass[ignoreframetext]{beamer} +\input{preamble.tex} + +\begin{document} +\mode +\input{seminar.tex} +\mode* + +\begin{frame}[allowframebreaks] + \printbibliography +\end{frame} +\end{document} diff --git a/modules/02-passwd/seminar.tex b/modules/02-passwd/seminar.tex new file mode 100644 index 0000000..b263d84 --- /dev/null +++ b/modules/02-passwd/seminar.tex @@ -0,0 +1,190 @@ +\title{% + How do you know it's secure? + Passwords +} +\author{Daniel Bosk\thanks{% + This material is available under the Creative Commons + Attribution-NonCommercial-ShareAlike (CC-BY-NC-SA) 4.0 international + license. + The material was written with some aid from GitHub Copilot. +}} +\institute{% + KTH EECS +} + +\begin{frame} + \maketitle +\end{frame} + +\mode* + +\begin{abstract} + \input{abstract.tex} +\end{abstract} + +\clearpage + +\section{Some interesting questions} + +\begin{frame} + \begin{question} + What part of security is not a science? + \end{question} +\end{frame} + +\section{The papers} + +\begin{frame} + \begin{itemize} + \item \fullcite{OfPasswordsAndPeople} + \item \fullcite{CanLongPasswordsBeSecureAndUsable} + \item \fullcite{WhyPhishingWorks} + \end{itemize} + \begin{onlyenv}<2> + \begin{remark} + Focus on the \emph{methods}, not the results! + \end{remark} + \end{onlyenv} +\end{frame} + +\section{Related to TAMOS} + +\begin{frame} + \begin{exercise} + \begin{itemize} + \item Try to relate the three papers + \begin{itemize} + \item \citetitle{OfPasswordsAndPeople} + \item \citetitle{CanLongPasswordsBeSecureAndUsable} + \item \citetitle{WhyPhishingWorks} + \end{itemize} + \item to the TAMOS lectures + \begin{itemize} + \item Scientific inferences, + \item Observation and measurements, + \item Experiments + \end{itemize} + \end{itemize} + \end{exercise} +\end{frame} + +\begin{frame} + \begin{question} + \begin{itemize} + \item What type of scientific inferences were they doing? + \end{itemize} + \end{question} + + \begin{question} + \begin{itemize} + \item What type of observation and measurement? + \begin{itemize} + \item Direct or indirect? Instruments? + \item Needed theory? + \end{itemize} + \end{itemize} + \end{question} + + \begin{question} + \begin{itemize} + \item Operationalization? + \begin{itemize} + \item What did they measure and how? + \item Measurement errors? + \end{itemize} + \end{itemize} + \end{question} +\end{frame} + +\begin{frame} + \begin{question} + \begin{itemize} + \item Did they do hypothesis testing? + \end{itemize} + \end{question} + + \begin{question} + \begin{itemize} + \item How was their experimental setup? Or was it observational? + \begin{itemize} + \item Randomize intervention and control? + \item Background factors? + \item Blinding? + \item Natural experiment? + \end{itemize} + \end{itemize} + \end{question} + + \begin{question} + \begin{itemize} + \item Error handling? + \begin{itemize} + \item Observer effect? + \end{itemize} + \end{itemize} + \end{question} +\end{frame} + +\section{More general} + +\begin{frame} + \begin{question} + \begin{itemize} + \item We had \citetitle{OfPasswordsAndPeople}, why do we need + \citetitle{CanLongPasswordsBeSecureAndUsable}? + \end{itemize} + \end{question} + + \begin{question} + \begin{itemize} + \item What can we say about the validity of these results? + \end{itemize} + \end{question} + + \begin{question} + \begin{itemize} + \item How well representative are the results? + \end{itemize} + \end{question} +\end{frame} + +\begin{frame} + \begin{question} + \begin{itemize} + \item What factors affect how users choose passwords? + \begin{itemize} + \item Is it just the password policy? + \item Can we know from these studies? + %\item We'd need more studies focusing on other factors, then a + %meta-analysis to see what factors are most important and how they + %interact. + \end{itemize} + \end{itemize} + \end{question} +\end{frame} + +\section{Preoceeding to Models} + +\begin{frame} + \begin{question} + Did they make use of any models? + \end{question} +\end{frame} + +\begin{frame} + \begin{example} + \begin{itemize} + \item Entropy as a model for password strength (guessability). + \begin{itemize} + \item However, later works show that it's not that good a model. + \end{itemize} + \end{itemize} + \end{example} + + \begin{remark} + \begin{itemize} + \item This is the topic of the next section of the course. + \end{itemize} + \end{remark} +\end{frame} + diff --git a/modules/02-passwd/slides.tex b/modules/02-passwd/slides.tex index 2287028..a77bbb7 100644 --- a/modules/02-passwd/slides.tex +++ b/modules/02-passwd/slides.tex @@ -1,100 +1,6 @@ \documentclass[ignoreframetext]{beamer} \input{preamble.tex} -\usetheme{Berlin} -\setbeamertemplate{footline}%{miniframes theme} -{% - \begin{beamercolorbox}[colsep=1.5pt]{upper separation line foot} - \end{beamercolorbox} - \begin{beamercolorbox}[ht=2.5ex,dp=1.125ex,% - leftskip=.3cm,rightskip=.3cm plus1fil]{author in head/foot}% - \leavevmode{\usebeamerfont{author in head/foot}\insertshortauthor}% - \hfill% - {\usebeamerfont{institute in head/foot}\usebeamercolor[fg]{institute in head/foot}\insertshortinstitute}% - \end{beamercolorbox}% - \begin{beamercolorbox}[ht=2.5ex,dp=1.125ex,% - leftskip=.3cm,rightskip=.3cm plus1fil]{title in head/foot}% - {\usebeamerfont{title in head/foot}\insertshorttitle} \hfill \insertframenumber% - \end{beamercolorbox}% - \begin{beamercolorbox}[colsep=1.5pt]{lower separation line foot} - \end{beamercolorbox} -} -\setbeamercovered{transparent} -\setbeamertemplate{bibliography item}[text] - -\AtBeginSection[]{% - \begin{frame} - \tableofcontents[currentsection] - \end{frame} -} - -\ProvideDocumentEnvironment{assumption}{o}{% - \IfValueTF{#1}{% - \begin{block}{Assumption: #1} - }{% - \begin{block}{Assumption} - } -}{% - \end{block} -} - -\ProvideDocumentEnvironment{protocol}{o}{% - \IfValueTF{#1}{% - \begin{block}{Protocol: #1} - }{% - \begin{block}{Protocol} - } -}{% - \end{block} -} - -\ProvideDocumentEnvironment{remark}{o}{% - \IfValueTF{#1}{% - \begin{alertblock}{Note: #1} - }{% - \begin{alertblock}{Note} - } -}{% - \end{alertblock} -} - -\ProvideDocumentEnvironment{idea}{o}{% - \IfValueTF{#1}{% - \begin{block}{Idea: #1} - }{% - \begin{block}{Idea} - } -}{% - \end{block} -} - -\ProvideDocumentEnvironment{question}{o}{% - \setbeamercolor{block body}{bg=orange!15,fg=black} - \setbeamercolor{block title}{bg=orange,fg=white} - \setbeamercolor{local structure}{fg=orange} - \IfValueTF{#1}{% - \begin{block}{Question: #1} - }{% - \begin{block}{Question} - } -}{% - \end{block} -} - -\ProvideDocumentEnvironment{exercise}{o}{% - \setbeamercolor{block body}{bg=yellow!10,fg=black} - \setbeamercolor{block title}{bg=yellow,fg=black} - \setbeamercolor{local structure}{fg=yellow} - \IfValueTF{#1}{% - \begin{block}{Exercise: #1} - }{% - \begin{block}{Exercise} - } -}{% - \end{block} -} - - \begin{document} \mode \input{contents.tex} diff --git a/modules/03-models/.gitignore b/modules/03-models/.gitignore new file mode 100644 index 0000000..3344cd5 --- /dev/null +++ b/modules/03-models/.gitignore @@ -0,0 +1,3 @@ +objectives.tex +*.csv +seminar-slides.pdf diff --git a/modules/03-models/Makefile b/modules/03-models/Makefile new file mode 100644 index 0000000..2ea7216 --- /dev/null +++ b/modules/03-models/Makefile @@ -0,0 +1,26 @@ +.PHONY: all +all: seminar-slides.pdf + +LATEXFLAGS+= -shell-escape + +SRC+= preamble.tex +SRC+= abstract.tex seminar.tex + +DEPENDS+= objectives.tex + +seminar-slides.pdf: seminar-slides.tex ${SRC} ${DEPENDS} + +objectives.tex: ../00-intro/objectives.tex + +${DEPENDS}: + ${LN} $< $@ + + +.PHONY: clean +clean: + ${RM} seminar-slides.pdf + ${RM} ${DEPENDS} + + +INCLUDE_MAKEFILES=../../makefiles +include ${INCLUDE_MAKEFILES}/tex.mk diff --git a/modules/03-models/extendoldsoft.md b/modules/03-models/extendoldsoft.md new file mode 100644 index 0000000..ae25e2a --- /dev/null +++ b/modules/03-models/extendoldsoft.md @@ -0,0 +1,32 @@ +--- +title: Exercise: Extending old software +authors: + - Arve Gengelbach +--- +The goal of this exercise is to harden an old network-connected banking +software that is checked to be working correctly on input that is well-formed, +however on any other input anything could happen from simple shutdown +(availability) to leak of confidential data. Programmers that could program in +the system's programming language are not around anymore. + +To increase the assurance level the idea of this exercise is to discussing +formal-methods based hardening in this particular scenario. + +The approach to harden the system is in two steps, first by adding a filtering +software that drops any malformed network traffic before this traffic could +even enter the possibly partially broken banking software, and second by +checking that the filter works correctly. Well-formed input can be specified by +a regular expression for this program. + +Tasks: + +1. Formulate a security property that could be verified for each of the two: + + a. the interface between (the output of) the filter and (the input of) the + old banking application + + b. the application that does the filtering + +2. Discuss how to verify the security property: Which formal method should be + used for which scenario? What are the reasons? State any assumptions on + input/output. diff --git a/modules/03-models/preamble.tex b/modules/03-models/preamble.tex new file mode 100644 index 0000000..43bbd69 --- /dev/null +++ b/modules/03-models/preamble.tex @@ -0,0 +1,56 @@ +\usepackage[utf8]{inputenc} +\usepackage[T1]{fontenc} +\usepackage[british]{babel} +\usepackage{booktabs} + +\usepackage[% + natbib, + citestyle=verbose,singletitle=false, + style=verbose, + maxbibnames=99,% + isbn=false,doi=false,url=true +]{biblatex} +\addbibresource{bibliography.bib} + +\usepackage[all]{foreign} +\renewcommand{\foreignfullfont}{} +\renewcommand{\foreignabbrfont}{} + +\usepackage{import} + +\usepackage[strict]{csquotes} +\SetCiteCommand{\autocite} +\usepackage[single]{acro} +\acsetup{cite/cmd={\autocite}} + +\usepackage[noend]{algpseudocode} +\usepackage{xparse} + +\let\email\texttt + +\usepackage[outputdir=ltxobj]{minted} +\setminted{autogobble} + +\usepackage{pythontex} +\setpythontexoutputdir{.} +\setpythontexworkingdir{..} + +\usepackage{amsmath} +\usepackage{amssymb} +\usepackage{mathtools} +\usepackage{amsthm} +\usepackage{thmtools} +%\usepackage[unq]{unique} +\DeclareMathOperator{\powerset}{\mathcal{P}} + +\usepackage[binary-units]{siunitx} + +\usepackage{adjustbox} +\usepackage{lipsum} +\usepackage{multicol} +\usepackage{changepage} + +\usepackage{didactic} +\usepackage[capitalize]{cleveref} + +\input{objectives.tex} diff --git a/modules/03-models/seminar-slides.tex b/modules/03-models/seminar-slides.tex new file mode 100644 index 0000000..3131b46 --- /dev/null +++ b/modules/03-models/seminar-slides.tex @@ -0,0 +1,12 @@ +\documentclass[ignoreframetext]{beamer} +\input{preamble.tex} + +\begin{document} +\mode +\input{seminar.tex} +\mode* + +\begin{frame}[allowframebreaks] + \printbibliography +\end{frame} +\end{document} diff --git a/modules/03-models/seminar.tex b/modules/03-models/seminar.tex new file mode 100644 index 0000000..ee9d621 --- /dev/null +++ b/modules/03-models/seminar.tex @@ -0,0 +1,62 @@ +\title{% + Models in security +} +\author{Daniel Bosk\thanks{% + This material is available under the Creative Commons + Attribution-NonCommercial-ShareAlike (CC-BY-NC-SA) 4.0 international + license. + The material was written with some aid from GitHub Copilot. +}} +\institute{% + KTH EECS +} + +\begin{frame} + \maketitle +\end{frame} + +\mode* + +\begin{abstract} + \input{abstract.tex} +\end{abstract} + +\clearpage + +\section{Related to TAMOS} + +\begin{frame} + \begin{question}[Models] + \begin{enumerate} + \item How do we use models in security? + \item How does that affect our results? + \begin{itemize} + \item Is ElGamal encryption secure or not? + \end{itemize} + \item What's the best we can do about the situation? + \end{enumerate} + \end{question} +\end{frame} + +\begin{frame} + \begin{question}[Engineering design] + \begin{enumerate} + \item Is our use of models just natural to the field or due to security + being an engineering discipline? + \end{enumerate} + \end{question} +\end{frame} + +\section{Model checking and theorem proving} + +\begin{frame} + \begin{question} + \begin{enumerate} + \item How do model checking and theorem proving relate to each other and + cryptography? + \item How sure can we be of the results when using model checking or + theorem proving? + Better than the ElGamal example? + \end{enumerate} + \end{question} +\end{frame} diff --git a/modules/03-models/teaching.md b/modules/03-models/teaching.md new file mode 100644 index 0000000..b62a6c2 --- /dev/null +++ b/modules/03-models/teaching.md @@ -0,0 +1,14 @@ +--- +title: Teaching material on formal methods +authors: + - Arve Gengelbach +--- +1. Lecture: Introduction to theorem proving +2. Lecture: Formal methods and interactive theorem proving +3. Lecture: Proving list-reverse correct (optional) +4. Lecture: Examples of what can be verified with interactive theorem provers +5. Lecture: Limitations of interactive theorem proving and conclusion +6. Article: Model checking: algorithmic verification and debugging +7. Lecture: Determining economic value of high assurance for commodity software + security +8. [Exercise: Extending old software](extendoldsoft.md) diff --git a/modules/04-anonymous/.gitignore b/modules/04-anonymous/.gitignore new file mode 100644 index 0000000..85d632d --- /dev/null +++ b/modules/04-anonymous/.gitignore @@ -0,0 +1,4 @@ +notes.pdf +slides.pdf +objectives.tex +literature diff --git a/modules/04-anonymous/Makefile b/modules/04-anonymous/Makefile new file mode 100644 index 0000000..eff3e29 --- /dev/null +++ b/modules/04-anonymous/Makefile @@ -0,0 +1,31 @@ +.PHONY: all +all: notes.pdf slides.pdf + +LATEXFLAGS+= -shell-escape + +SRC+= preamble.tex +SRC+= abstract.tex contents.tex + +DEPENDS+= objectives.tex + +notes.pdf: notes.tex +notes.pdf: ${SRC} ${DEPENDS} ${FIGS} + +slides.pdf: slides.tex +slides.pdf: ${SRC} ${DEPENDS} ${FIGS} + +objectives.tex: ../00-intro/objectives.tex + +${DEPENDS}: + ${LN} $< $@ + + +.PHONY: clean +clean: + ${RM} notes.pdf slides.pdf + ${RM} ${DEPENDS} + ${MAKE} -C fig clean + + +INCLUDE_MAKEFILES=../../makefiles +include ${INCLUDE_MAKEFILES}/tex.mk diff --git a/modules/04-anonymous/abstract.tex b/modules/04-anonymous/abstract.tex new file mode 100644 index 0000000..ae4a777 --- /dev/null +++ b/modules/04-anonymous/abstract.tex @@ -0,0 +1,30 @@ +% What's the problem? +% Why is it a problem? Research gap left by other approaches? +% Why is it important? Why care? +% What's the approach? How to solve the problem? +% What's the findings? How was it evaluated, what are the results, limitations, +% what remains to be done? + +% XXX Summary +\emph{Summary:} +In this assignment we will explore the research methods used in the area of +anonymous communication systems, such as Tor. +This is interesting because it requires a wide range of methods. + +% XXX Motivation and intended learning outcomes +\emph{Intended learning outcomes:} +This assignment focuses on practice to +\begin{itemize} + \item \LOrelate; + \item \LOevaluate; + \item \LOapply; + \item \LOcomm. +\end{itemize} + +% XXX Prerequisites +\emph{Prerequisites:} +We need basic knowledge of security, corresponding to an introductory course in +the subject. +We also need a high-level overview of the breadth of research methods used in +the area of security, corresponding to the overview lecture \enquote{The +Scientific Method} given in the course. diff --git a/modules/04-anonymous/bibliography.bib b/modules/04-anonymous/bibliography.bib new file mode 100644 index 0000000..e69de29 diff --git a/modules/04-anonymous/contents.tex b/modules/04-anonymous/contents.tex new file mode 100644 index 0000000..2bc200c --- /dev/null +++ b/modules/04-anonymous/contents.tex @@ -0,0 +1,115 @@ +\title{% + How do you know it's secure? + Anonymous communication +} +\author{Daniel Bosk\thanks{% + This material is available under the Creative Commons + Attribution-NonCommercial-ShareAlike (CC-BY-NC-SA) 4.0 international + license. + The material was written with some aid from GitHub Copilot. +}} +\institute{% + KTH EECS +} + +\begin{frame} + \maketitle +\end{frame} + +\mode* + +\begin{abstract} + \input{abstract.tex} +\end{abstract} + +\clearpage + +\section{Users get routed} + +\begin{frame} + \begin{question}[Users get routed] + \begin{itemize} + \item What was the research question? + \item What methods did they use? + \item Why is that a good method? + \item Can we use another method? + \end{itemize} + \end{question} +\end{frame} + +\begin{frame} + \begin{question} + \begin{itemize} + \item Why do we need to use empirical methods in this case? + \item What effect does data have on validity? + \end{itemize} + \end{question} +\end{frame} + +\section{Shadow: Running Tor in a box} + +\begin{frame} + \begin{question}[Shadow: Running Tor in a box] + \begin{itemize} + \item What was the research question? + \item What methods did they use? + \item Why is that a good method? + \item Can we use another method? + \end{itemize} + \end{question} +\end{frame} + +\begin{frame} + \begin{question} + \begin{itemize} + \item They evaluate the quality of a research tool. + \item What are the challenges in doing that? + \end{itemize} + \end{question} +\end{frame} + +\section{WF with Website Oracles} + +\begin{frame} + \begin{question}[WF with Website Oracles] + \begin{itemize} + \item What was the research question? + \item What methods did they use? + \item Why is that a good method? + \item Can we use another method? + \end{itemize} + \end{question} +\end{frame} + +\begin{frame} + \begin{question} + \begin{itemize} + \item What type of contribution do we get from this paper? + What is the Website Oracle? + \item How do they show WOs exist in reality? + \end{itemize} + \end{question} +\end{frame} + +\section{Evaluating WF in Real World} + +\begin{frame} + \begin{question}[Evaluating WF in Real World] + \begin{itemize} + \item What was the research question? + \item What methods did they use? + \item Why is that a good method? + \item Can we use another method? + \end{itemize} + \end{question} +\end{frame} + +\begin{frame} + \begin{question} + \begin{itemize} + \item They complained about the evaluation of WF in other papers. + What was the problem? + \item Are there any problems with this paper's approach? + \end{itemize} + \end{question} +\end{frame} diff --git a/modules/04-anonymous/fig/.gitignore b/modules/04-anonymous/fig/.gitignore new file mode 100644 index 0000000..2bc47a4 --- /dev/null +++ b/modules/04-anonymous/fig/.gitignore @@ -0,0 +1,2 @@ +password_reuse.png +password_strength.png diff --git a/modules/04-anonymous/fig/Makefile b/modules/04-anonymous/fig/Makefile new file mode 100644 index 0000000..ea16ce9 --- /dev/null +++ b/modules/04-anonymous/fig/Makefile @@ -0,0 +1,5 @@ +.PHONY: all +all: + +.PHONY: clean +clean: diff --git a/course-design/notes.tex b/modules/04-anonymous/notes.tex similarity index 57% rename from course-design/notes.tex rename to modules/04-anonymous/notes.tex index 8a8020d..49b372d 100644 --- a/course-design/notes.tex +++ b/modules/04-anonymous/notes.tex @@ -38,32 +38,27 @@ \let\subsection\section \let\section\chapter -\newsubfloat{figure}% Allow subfloats in figure environment -\usepackage{fullwidth} - \input{preamble.tex} \usepackage[noamsthm,notheorems]{beamerarticle} \setjobnamebeamerversion{slides} -\usepackage[inline]{enumitem} - -\usepackage{authblk} -\let\institute\affil +%\usepackage{authblk} +%\let\institute\affil -\declaretheorem[numbered=unless unique,style=theorem]{theorem} -\declaretheorem[numbered=unless unique,style=definition]{definition} -\declaretheorem[numbered=unless unique,style=definition]{assumption} -\declaretheorem[numbered=unless unique,style=definition]{protocol} -\declaretheorem[numbered=unless unique,style=example]{example} -%\declaretheorem[style=definition,numbered=unless unique, -% name=Example,refname={example,examples}]{example} -\declaretheorem[numbered=unless unique,style=remark]{remark} -\declaretheorem[numbered=unless unique,style=remark]{idea} -\declaretheorem[numbered=unless unique,style=exercise]{exercise} -\declaretheorem[numbered=unless unique,style=exercise]{question} -\declaretheorem[numbered=unless unique,style=solution]{solution} +\declaretheorem[style=theorem]{theorem} +\declaretheorem[style=definition]{definition} +\declaretheorem[style=definition]{assumption} +\declaretheorem[style=definition]{protocol} +\declaretheorem[style=example]{example} +\declaretheorem[style=remark]{remark} +\declaretheorem[style=remark]{idea} +\declaretheorem[style=exercise]{exercise} +\declaretheorem[style=exercise]{question} +\declaretheorem[style=solution]{solution} \begin{document} \input{contents.tex} + +\printbibliography \end{document} diff --git a/modules/04-anonymous/preamble.tex b/modules/04-anonymous/preamble.tex new file mode 100644 index 0000000..dce955a --- /dev/null +++ b/modules/04-anonymous/preamble.tex @@ -0,0 +1,55 @@ +\usepackage[utf8]{inputenc} +\usepackage[T1]{fontenc} +\usepackage[british]{babel} +\usepackage{booktabs} + +\usepackage[% + natbib, + citestyle=verbose,singletitle=false, + style=verbose, + maxbibnames=99,% + isbn=false,doi=false,url=true +]{biblatex} +\addbibresource{bibliography.bib} + +\usepackage[all]{foreign} +\renewcommand{\foreignfullfont}{} +\renewcommand{\foreignabbrfont}{} + +\usepackage{import} + +\usepackage[strict]{csquotes} +\SetCiteCommand{\autocite} +\usepackage[single]{acro} +\acsetup{cite/cmd={\autocite}} + +\usepackage[noend]{algpseudocode} +\usepackage{xparse} + +\let\email\texttt + +\usepackage[outputdir=ltxobj]{minted} +\setminted{autogobble} + +\usepackage{pythontex} +\setpythontexoutputdir{.} +\setpythontexworkingdir{..} + +\usepackage{amsmath} +\usepackage{amssymb} +\usepackage{mathtools} +\usepackage{amsthm} +\usepackage{thmtools} +%\usepackage[unq]{unique} +\DeclareMathOperator{\powerset}{\mathcal{P}} + +\usepackage[binary-units]{siunitx} + +\usepackage{adjustbox} +\usepackage{lipsum} +\usepackage{multicol} +\usepackage{changepage} + +\usepackage[capitalize]{cleveref} + +\input{objectives.tex} diff --git a/modules/04-anonymous/seminar.md b/modules/04-anonymous/seminar.md new file mode 100644 index 0000000..cf03b91 --- /dev/null +++ b/modules/04-anonymous/seminar.md @@ -0,0 +1,9 @@ +We will talk about some methods used in researching Tor, this covers a variety +of methods ranging from deductive to empirical experiments to simulations. + +**Participation**: online in the class Zoom room. + +**Preparation**: The material in this module that precedes this page. + +**Seminar content**: Briefly discuss the methods and research questions. + diff --git a/course-design/slides.tex b/modules/04-anonymous/slides.tex similarity index 95% rename from course-design/slides.tex rename to modules/04-anonymous/slides.tex index 2fbc768..395cdf5 100644 --- a/course-design/slides.tex +++ b/modules/04-anonymous/slides.tex @@ -20,8 +20,7 @@ \end{beamercolorbox} } \setbeamercovered{transparent} -%\setbeamertemplate{bibliography item}[text] -\setbeamertemplate{bibliography item}{} +\setbeamertemplate{bibliography item}{\relax} \AtBeginSection[]{% \begin{frame} @@ -100,4 +99,8 @@ \mode \input{contents.tex} \mode* + +\begin{frame}[allowframebreaks] + \printbibliography +\end{frame} \end{document} diff --git a/modules/04-quantqual/.gitignore b/modules/04-quantqual/.gitignore new file mode 100644 index 0000000..85d632d --- /dev/null +++ b/modules/04-quantqual/.gitignore @@ -0,0 +1,4 @@ +notes.pdf +slides.pdf +objectives.tex +literature diff --git a/modules/04-quantqual/Makefile b/modules/04-quantqual/Makefile new file mode 100644 index 0000000..eff3e29 --- /dev/null +++ b/modules/04-quantqual/Makefile @@ -0,0 +1,31 @@ +.PHONY: all +all: notes.pdf slides.pdf + +LATEXFLAGS+= -shell-escape + +SRC+= preamble.tex +SRC+= abstract.tex contents.tex + +DEPENDS+= objectives.tex + +notes.pdf: notes.tex +notes.pdf: ${SRC} ${DEPENDS} ${FIGS} + +slides.pdf: slides.tex +slides.pdf: ${SRC} ${DEPENDS} ${FIGS} + +objectives.tex: ../00-intro/objectives.tex + +${DEPENDS}: + ${LN} $< $@ + + +.PHONY: clean +clean: + ${RM} notes.pdf slides.pdf + ${RM} ${DEPENDS} + ${MAKE} -C fig clean + + +INCLUDE_MAKEFILES=../../makefiles +include ${INCLUDE_MAKEFILES}/tex.mk diff --git a/modules/04-quantqual/abstract.tex b/modules/04-quantqual/abstract.tex new file mode 100644 index 0000000..2b33cc5 --- /dev/null +++ b/modules/04-quantqual/abstract.tex @@ -0,0 +1,31 @@ +% What's the problem? +% Why is it a problem? Research gap left by other approaches? +% Why is it important? Why care? +% What's the approach? How to solve the problem? +% What's the findings? How was it evaluated, what are the results, limitations, +% what remains to be done? + +% XXX Summary +\emph{Summary:} +In this seminar we will explore the relationship between quantitative and +qualitative methods. +We're particularly interested in how they complement each other and why +\emph{qualitative} methods are so important in cybersecurity. + +% XXX Motivation and intended learning outcomes +\emph{Intended learning outcomes:} +This assignment focuses on practice to +\begin{itemize} + \item \LOrelate; + \item \LOevaluate; + \item \LOapply; + \item \LOcomm. +\end{itemize} + +% XXX Prerequisites +\emph{Prerequisites:} +We need basic knowledge of security, corresponding to an introductory course in +the subject. +We also need a high-level overview of the breadth of research methods used in +the area of security, corresponding to the overview lecture \enquote{The +Scientific Method} given in the course. diff --git a/modules/04-quantqual/bibliography.bib b/modules/04-quantqual/bibliography.bib new file mode 100644 index 0000000..e69de29 diff --git a/modules/04-quantqual/contents.tex b/modules/04-quantqual/contents.tex new file mode 100644 index 0000000..dc1822f --- /dev/null +++ b/modules/04-quantqual/contents.tex @@ -0,0 +1,235 @@ +\title{% + Qualitative and quantitative methods +} +\author{Daniel Bosk\thanks{% + This material is available under the Creative Commons + Attribution-NonCommercial-ShareAlike (CC-BY-NC-SA) 4.0 international + license. + The material was written with some aid from GitHub Copilot. +}} +\institute{% + KTH EECS +} + +\begin{frame} + \maketitle +\end{frame} + +\mode* + +\begin{abstract} + \input{abstract.tex} +\end{abstract} + +\clearpage + +\section{Warm-up} + +\begin{frame} +\begin{columns}[T,totalwidth=\linewidth] + \begin{column}{0.5\linewidth} + \begin{block}{Quantitative} + \begin{itemize} + \item Shadow: Running Tor in a box + \item Website Fingerprinting with Website Oracles + \item Evaluating the Real-World Applicability of WF + \item Dos and Don'ts of Machine Learning in Security + \end{itemize} + \end{block} + \end{column} + \begin{column}{0.5\linewidth} + \begin{block}{Qualitative} + \begin{itemize} + \item Why phishing works + \item Why Johnny can't encrypt + \item Comparing the Usability of Cryptographic APIs + \end{itemize} + \end{block} + \end{column} +\end{columns} + \begin{exercise}[Venting] + \begin{itemize} + \item What are your thoughts after reading the papers? + \end{itemize} + \end{exercise} +\end{frame} + +\begin{frame} + \begin{question} + \begin{itemize} + \item How do qualitative and quantitative methods complement each + other? + \end{itemize} + \end{question} +\end{frame} + +\section{Quantitative methods} + +\begin{frame} + \begin{block}{Papers} + \begin{itemize} + \item Shadow: Running Tor in a box + \item Website Fingerprinting with Website Oracles + \item Online Website Fingerprinting: Evaluating the Real-World + Applicability of Website Fingerprinting + \item Dos and Don'ts of Machine Learning in Security + \end{itemize} + \end{block} + + \begin{question}[Venting] + \begin{itemize} + \item What are your thoughts after reading the papers? + \end{itemize} + \end{question} +\end{frame} + +%\section{Users get routed} +% +%\begin{frame} +% \begin{question}[Users get routed] +% \begin{itemize} +% \item What was the research question? +% \item What methods did they use? +% \item Why is that a good method? +% \item Can we use another method? +% \end{itemize} +% \end{question} +%\end{frame} +% +%\begin{frame} +% \begin{question} +% \begin{itemize} +% \item Why do we need to use empirical methods in this case? +% \item What effect does data have on validity? +% \end{itemize} +% \end{question} +%\end{frame} + +\subsection{Shadow: Running Tor in a box} + +\begin{frame} + \begin{question}[Shadow: Running Tor in a box] + \begin{itemize} + \item What was the research question? + \item What methods did they use? + \item Why is that a good method? + \item Can we use another method? + \end{itemize} + \end{question} +\end{frame} + +\begin{frame} + \begin{question} + \begin{itemize} + \item They evaluate the quality of a research tool. + \item What are the challenges in doing that? + \end{itemize} + \end{question} +\end{frame} + +\subsection{WF with Website Oracles} + +\begin{frame} + \begin{question}[WF with Website Oracles] + \begin{itemize} + \item What was the research question? + \item What methods did they use? + \item Why is that a good method? + \item Can we use another method? + \end{itemize} + \end{question} +\end{frame} + +\begin{frame} + \begin{question} + \begin{itemize} + \item What type of contribution do we get from this paper? + What is the Website Oracle? + \item How do they show WOs exist in reality? + \end{itemize} + \end{question} +\end{frame} + +\subsection{Evaluating WF in Real World} + +\begin{frame} + \begin{question}[Evaluating WF in Real World] + \begin{itemize} + \item What was the research question? + \item What methods did they use? + \item Why is that a good method? + \item Can we use another method? + \end{itemize} + \end{question} +\end{frame} + +\begin{frame} + \begin{question} + \begin{itemize} + \item They complained about the evaluation of WF in other papers. + What was the problem? + \item Are there any problems with this paper's approach? + \end{itemize} + \end{question} +\end{frame} + + +\section{Qualitative methods} + +\begin{frame} + \begin{block}{Papers} + \begin{itemize} + \item Why phishing works + \item Why Johnny can't encrypt + \item Comparing the Usability of Cryptographic APIs + \end{itemize} + \end{block} + + \begin{question}[Venting] + \begin{itemize} + \item What are your thoughts after reading the papers? + \end{itemize} + \end{question} +\end{frame} + +\subsection{Why phishing works} + +\begin{frame} + \begin{question}[Evaluating WF in Real World] + \begin{itemize} + \item What was the research question? + \item What methods did they use? + \item Why is that a good method? + \item Can we use another method? + \end{itemize} + \end{question} +\end{frame} + +\subsection{Why Johnny can't encrypt} + +\begin{frame} + \begin{question}[Evaluating WF in Real World] + \begin{itemize} + \item What was the research question? + \item What methods did they use? + \item Why is that a good method? + \item Can we use another method? + \end{itemize} + \end{question} +\end{frame} + +\subsection{Comparing the Usability of Cryptographic APIs} + +\begin{frame} + \begin{question}[Evaluating WF in Real World] + \begin{itemize} + \item What was the research question? + \item What methods did they use? + \item Why is that a good method? + \item Can we use another method? + \end{itemize} + \end{question} +\end{frame} + + + diff --git a/modules/04-quantqual/notes.tex b/modules/04-quantqual/notes.tex new file mode 100644 index 0000000..49b372d --- /dev/null +++ b/modules/04-quantqual/notes.tex @@ -0,0 +1,64 @@ +\documentclass[a4paper,10pt,article,oneside]{memoir} +%%% Tufte %%% +\usepackage{marginfix} +%\setlength{\evensidemargin}{\oddsidemargin} +\marginparmargin{outer} +\setlrmarginsandblock{2.5cm}{8cm}{*} + +\footnotesinmargin + +\usepackage{ragged2e} +\renewcommand{\sidefootform}{\RaggedRight} +\renewcommand{\foottextfont}{\footnotesize\RaggedRight} + +\setmpjustification{\RaggedRight}{\RaggedRight} + +% margin figure and caption typeset ragged against text block +\setfloatadjustment{marginfigure}{\mpjustification} +\setmarginfloatcaptionadjustment{figure}{\captionstyle{\mpjustification}} + +% From https://tex.stackexchange.com/a/324757/17418 +% Palatino for main text and math +\usepackage[osf,sc]{mathpazo} + +% Helvetica for sans serif +% (scaled to match size of Palatino) +\usepackage[scaled=0.90]{helvet} + +% Bera Mono for monospaced +% (scaled to match size of Palatino) +\usepackage[scaled=0.85]{beramono} + +\setlxvchars\setxlvchars +\checkandfixthelayout + +\nouppercaseheads +%%% end tufte %%% +\let\subsubsection\subsection +\let\subsection\section +\let\section\chapter + +\input{preamble.tex} + +\usepackage[noamsthm,notheorems]{beamerarticle} +\setjobnamebeamerversion{slides} + +%\usepackage{authblk} +%\let\institute\affil + +\declaretheorem[style=theorem]{theorem} +\declaretheorem[style=definition]{definition} +\declaretheorem[style=definition]{assumption} +\declaretheorem[style=definition]{protocol} +\declaretheorem[style=example]{example} +\declaretheorem[style=remark]{remark} +\declaretheorem[style=remark]{idea} +\declaretheorem[style=exercise]{exercise} +\declaretheorem[style=exercise]{question} +\declaretheorem[style=solution]{solution} + +\begin{document} +\input{contents.tex} + +\printbibliography +\end{document} diff --git a/modules/04-quantqual/preamble.tex b/modules/04-quantqual/preamble.tex new file mode 100644 index 0000000..dce955a --- /dev/null +++ b/modules/04-quantqual/preamble.tex @@ -0,0 +1,55 @@ +\usepackage[utf8]{inputenc} +\usepackage[T1]{fontenc} +\usepackage[british]{babel} +\usepackage{booktabs} + +\usepackage[% + natbib, + citestyle=verbose,singletitle=false, + style=verbose, + maxbibnames=99,% + isbn=false,doi=false,url=true +]{biblatex} +\addbibresource{bibliography.bib} + +\usepackage[all]{foreign} +\renewcommand{\foreignfullfont}{} +\renewcommand{\foreignabbrfont}{} + +\usepackage{import} + +\usepackage[strict]{csquotes} +\SetCiteCommand{\autocite} +\usepackage[single]{acro} +\acsetup{cite/cmd={\autocite}} + +\usepackage[noend]{algpseudocode} +\usepackage{xparse} + +\let\email\texttt + +\usepackage[outputdir=ltxobj]{minted} +\setminted{autogobble} + +\usepackage{pythontex} +\setpythontexoutputdir{.} +\setpythontexworkingdir{..} + +\usepackage{amsmath} +\usepackage{amssymb} +\usepackage{mathtools} +\usepackage{amsthm} +\usepackage{thmtools} +%\usepackage[unq]{unique} +\DeclareMathOperator{\powerset}{\mathcal{P}} + +\usepackage[binary-units]{siunitx} + +\usepackage{adjustbox} +\usepackage{lipsum} +\usepackage{multicol} +\usepackage{changepage} + +\usepackage[capitalize]{cleveref} + +\input{objectives.tex} diff --git a/modules/04-quantqual/slides.tex b/modules/04-quantqual/slides.tex new file mode 100644 index 0000000..395cdf5 --- /dev/null +++ b/modules/04-quantqual/slides.tex @@ -0,0 +1,106 @@ +\documentclass[ignoreframetext]{beamer} +\input{preamble.tex} + +\usetheme{Berlin} +\setbeamertemplate{footline}%{miniframes theme} +{% + \begin{beamercolorbox}[colsep=1.5pt]{upper separation line foot} + \end{beamercolorbox} + \begin{beamercolorbox}[ht=2.5ex,dp=1.125ex,% + leftskip=.3cm,rightskip=.3cm plus1fil]{author in head/foot}% + \leavevmode{\usebeamerfont{author in head/foot}\insertshortauthor}% + \hfill% + {\usebeamerfont{institute in head/foot}\usebeamercolor[fg]{institute in head/foot}\insertshortinstitute}% + \end{beamercolorbox}% + \begin{beamercolorbox}[ht=2.5ex,dp=1.125ex,% + leftskip=.3cm,rightskip=.3cm plus1fil]{title in head/foot}% + {\usebeamerfont{title in head/foot}\insertshorttitle} \hfill \insertframenumber% + \end{beamercolorbox}% + \begin{beamercolorbox}[colsep=1.5pt]{lower separation line foot} + \end{beamercolorbox} +} +\setbeamercovered{transparent} +\setbeamertemplate{bibliography item}{\relax} + +\AtBeginSection[]{% + \begin{frame} + \tableofcontents[currentsection] + \end{frame} +} + +\ProvideDocumentEnvironment{assumption}{o}{% + \IfValueTF{#1}{% + \begin{block}{Assumption: #1} + }{% + \begin{block}{Assumption} + } +}{% + \end{block} +} + +\ProvideDocumentEnvironment{protocol}{o}{% + \IfValueTF{#1}{% + \begin{block}{Protocol: #1} + }{% + \begin{block}{Protocol} + } +}{% + \end{block} +} + +\ProvideDocumentEnvironment{remark}{o}{% + \IfValueTF{#1}{% + \begin{alertblock}{Note: #1} + }{% + \begin{alertblock}{Note} + } +}{% + \end{alertblock} +} + +\ProvideDocumentEnvironment{idea}{o}{% + \IfValueTF{#1}{% + \begin{block}{Idea: #1} + }{% + \begin{block}{Idea} + } +}{% + \end{block} +} + +\ProvideDocumentEnvironment{question}{o}{% + \setbeamercolor{block body}{bg=orange!15,fg=black} + \setbeamercolor{block title}{bg=orange,fg=white} + \setbeamercolor{local structure}{fg=orange} + \IfValueTF{#1}{% + \begin{block}{Question: #1} + }{% + \begin{block}{Question} + } +}{% + \end{block} +} + +\ProvideDocumentEnvironment{exercise}{o}{% + \setbeamercolor{block body}{bg=yellow!10,fg=black} + \setbeamercolor{block title}{bg=yellow,fg=black} + \setbeamercolor{local structure}{fg=yellow} + \IfValueTF{#1}{% + \begin{block}{Exercise: #1} + }{% + \begin{block}{Exercise} + } +}{% + \end{block} +} + + +\begin{document} +\mode +\input{contents.tex} +\mode* + +\begin{frame}[allowframebreaks] + \printbibliography +\end{frame} +\end{document} diff --git a/modules/05-scientificity/.gitignore b/modules/05-scientificity/.gitignore new file mode 100644 index 0000000..701f4cf --- /dev/null +++ b/modules/05-scientificity/.gitignore @@ -0,0 +1,5 @@ +notes.pdf +slides.pdf +objectives.tex +literature +seminar-slides.pdf diff --git a/modules/05-scientificity/Makefile b/modules/05-scientificity/Makefile new file mode 100644 index 0000000..37bdd51 --- /dev/null +++ b/modules/05-scientificity/Makefile @@ -0,0 +1,34 @@ +.PHONY: all +all: notes.pdf slides.pdf seminar-slides.pdf + +LATEXFLAGS+= -shell-escape + +SRC+= preamble.tex +SRC+= abstract.tex contents.tex + +DEPENDS+= objectives.tex + +notes.pdf: notes.tex +notes.pdf: ${SRC} ${DEPENDS} ${FIGS} + +slides.pdf: slides.tex +slides.pdf: ${SRC} ${DEPENDS} ${FIGS} + +seminar-slides.pdf: seminar-slides.tex seminar.tex ${SRC} ${DEPENDS} ${FIGS} + +objectives.tex: ../00-intro/objectives.tex + +${DEPENDS}: + ${LN} $< $@ + + +.PHONY: clean +clean: + ${RM} notes.pdf slides.pdf + ${RM} ${DEPENDS} + ${MAKE} -C fig clean + ${RM} seminar-slides.pdf + + +INCLUDE_MAKEFILES=../../makefiles +include ${INCLUDE_MAKEFILES}/tex.mk diff --git a/modules/05-scientificity/abstract.tex b/modules/05-scientificity/abstract.tex new file mode 100644 index 0000000..b2a93c6 --- /dev/null +++ b/modules/05-scientificity/abstract.tex @@ -0,0 +1,28 @@ +% What's the problem? +% Why is it a problem? Research gap left by other approaches? +% Why is it important? Why care? +% What's the approach? How to solve the problem? +% What's the findings? How was it evaluated, what are the results, limitations, +% what remains to be done? + +% XXX Summary +\emph{Summary:} +In this module we will try to overview the methodological state of security. + +% XXX Motivation and intended learning outcomes +\emph{Intended learning outcomes:} +This assignment focuses on practice to +\begin{itemize} + \item \LOrelate; + \item \LOevaluate; + \item \LOapply; + \item \LOcomm. +\end{itemize} + +% XXX Prerequisites +\emph{Prerequisites:} +We need basic knowledge of security, corresponding to an introductory course in +the subject. +We also need a high-level overview of the breadth of research methods used in +the area of security, corresponding to the overview lecture \enquote{The +Scientific Method} given in the course. diff --git a/modules/05-scientificity/bibliography.bib b/modules/05-scientificity/bibliography.bib new file mode 100644 index 0000000..f25495c --- /dev/null +++ b/modules/05-scientificity/bibliography.bib @@ -0,0 +1,39 @@ +@article{HowToDesignComputerSecurityExperiments, + title = {How to design computer security experiments}, + author = {Peisert, Sean and Bishop, Matt}, + booktitle = {Fifth World Conference on Information Security Education, Proceedings of the IFIP TC11 WG 11.8, WISE 5, 19 to 21 June 2007, United States Military Academy, West Point, New York, USA}, + doi = {10.1007/978-0-387-73269-5_19}, + editor = {Futcher, Lynn and Dodge, Ronald}, + isbn = {978-0387732688}, + issn = {1868-4238}, + journal = {WISE}, + month = {6}, + pages = {141--148}, + publisher = {Springer Science+Business Media}, + url = {https://link.springer.com/content/pdf/10.1007%2F978-0-387-73269-5_19.pdf}, + volume = {237}, + year = {2007}, +} + +@inproceedings{DosAndDontsML, + title = {Dos and don'ts of machine learning in computer security}, + author = {Arp, Daniel and Quiring, Erwin and Pendlebury, Feargus and Warnecke, Alexander and Pierazzi, Fabio and Wressnegger, Christian and Cavallaro, Lorenzo and Rieck, Konrad}, + booktitle = {31st USENIX Security Symposium, USENIX Security 2022, Boston, MA, USA, August 10-12, 2022}, + editor = {Butler, Kevin R. B. and Thomas, Kurt}, + publisher = {USENIX Association}, + year = {2022}, +} + +@inproceedings{SoKScienceOfSecurity, + title = {SoK: Science, Security, and the Elusive Goal of Security as a +Scientific Pursuit}, + author = {Herley, Cormac and Van Oorschot, Paul C.}, + booktitle = {2017 IEEE Symposium on Security and Privacy, SP 2017, San Jose, CA, USA, May 22-26, 2017}, + doi = {10.1109/sp.2017.38}, + journal = {IEEE Symposium on Security and Privacy}, + month = {5}, + pages = {99--120}, + publisher = {IEEE Computer Society}, + url = {https://doi.org/10.1109/sp.2017.38}, + year = {2017}, +} diff --git a/modules/05-scientificity/contents.tex b/modules/05-scientificity/contents.tex new file mode 100644 index 0000000..9deeae7 --- /dev/null +++ b/modules/05-scientificity/contents.tex @@ -0,0 +1,76 @@ +\title{% + Scientificity in Security +} +\author{Daniel Bosk\thanks{% + This material is available under the Creative Commons + Attribution-NonCommercial-ShareAlike (CC-BY-NC-SA) 4.0 international + license. + GitHub Copilot was used to autocomplete parts of the source code for this + document. +}} +\institute{% + KTH EECS +} + +\begin{frame} + \maketitle +\end{frame} + +\mode* + +\begin{abstract} + \input{abstract.tex} +\end{abstract} + +\clearpage + + +\section{How to design computer security experiments} + +\Textcite{HowToDesignComputerSecurityExperiments} gives a good overview of how +to design computer security experiments. +\begin{frame} + \begin{question} + But is that applicable in all parts of security research? + \end{question} +\end{frame} + +\section{Dos and don'ts of machine learning in security} + +\Textcite{DosAndDontsML} discusses how to and how not to use machine learning +in security. + +\begin{frame} + \begin{question} + Based on this paper, what can we say about the use of machine learning in + the website fingerprinting papers? + \end{question} +\end{frame} + +\begin{frame} + \begin{question} + What can we actually say from these results, based on their method? + \end{question} +\end{frame} + +\section{Security as a scientific pursuit} + +\Textcite{SoKScienceOfSecurity} discusses security as a scientific pursuit. + +\begin{frame} + \begin{question} + How did they know that this is so? + What can we actually say from these results? + \end{question} +\end{frame} + +\begin{frame} + \begin{question} + What assurances can empirical experiments give about security? + \end{question} + + \begin{question} + What assurances can deductive reasoning give about security? + \end{question} +\end{frame} + diff --git a/modules/05-scientificity/fig/Makefile b/modules/05-scientificity/fig/Makefile new file mode 100644 index 0000000..ea16ce9 --- /dev/null +++ b/modules/05-scientificity/fig/Makefile @@ -0,0 +1,5 @@ +.PHONY: all +all: + +.PHONY: clean +clean: diff --git a/modules/05-scientificity/notes.tex b/modules/05-scientificity/notes.tex new file mode 100644 index 0000000..20e181f --- /dev/null +++ b/modules/05-scientificity/notes.tex @@ -0,0 +1,18 @@ +\documentclass[a4paper,10pt,article,oneside]{memoir} +\let\subsubsection\subsection +\let\subsection\section +\let\section\chapter + +\input{preamble.tex} + +\usepackage[noamsthm,notheorems]{beamerarticle} +\setjobnamebeamerversion{slides} + +%\usepackage{authblk} +%\let\institute\affil + +\begin{document} +\input{contents.tex} + +\printbibliography +\end{document} diff --git a/modules/05-scientificity/preamble.tex b/modules/05-scientificity/preamble.tex new file mode 100644 index 0000000..551ffc4 --- /dev/null +++ b/modules/05-scientificity/preamble.tex @@ -0,0 +1,57 @@ +\usepackage[utf8]{inputenc} +\usepackage[T1]{fontenc} +\usepackage[british]{babel} +\usepackage{booktabs} + +\usepackage[% + natbib, + citestyle=verbose,singletitle=false, + style=verbose, + maxbibnames=99,% + isbn=false,doi=false,url=true +]{biblatex} +\addbibresource{bibliography.bib} + +\usepackage[all]{foreign} +\renewcommand{\foreignfullfont}{} +\renewcommand{\foreignabbrfont}{} + +\usepackage{import} + +\usepackage[strict]{csquotes} +\SetCiteCommand{\autocite} +\usepackage[single]{acro} +\acsetup{cite/cmd={\autocite}} + +\usepackage[noend]{algpseudocode} +\usepackage{xparse} + +\let\email\texttt + +\usepackage[outputdir=ltxobj]{minted} +\setminted{autogobble} + +\usepackage{pythontex} +\setpythontexoutputdir{.} +\setpythontexworkingdir{..} + +\usepackage{amsmath} +\usepackage{amssymb} +\usepackage{mathtools} +\usepackage{amsthm} +\usepackage{thmtools} +%\usepackage[unq]{unique} +\DeclareMathOperator{\powerset}{\mathcal{P}} + +\usepackage[binary-units]{siunitx} + +\usepackage{adjustbox} +\usepackage{lipsum} +\usepackage{multicol} +\usepackage{changepage} + +\usepackage{didactic} + +\usepackage[capitalize]{cleveref} + +\input{objectives.tex} diff --git a/modules/05-scientificity/reflection.md b/modules/05-scientificity/reflection.md new file mode 100644 index 0000000..22bec86 --- /dev/null +++ b/modules/05-scientificity/reflection.md @@ -0,0 +1,98 @@ +# Introduction + +Let's think a bit about how we know what we teach in the security courses are +good things or not. This will let us combine all the classes of methods that +we've covered during the course. Remember also what they wrote in Security as a +Scientific Pursuit: we wouldn't want to just make tautological restatements of +our assumptions. + +Before the seminar, pick THREE (3) of the topics below. For each one you +picked, outline a methodology for how to evaluate it. + +## Passwords + +To do password-based authentication, we're taught that we should salt +and hash the passwords. The user submits the password to the server, +the server adds a 128-bit salt (random number) and hashes the password +using SHA256, then hashes the hash, and hashes that hash, and so on for 1000 +times. That final hash is used to check if it's the same as the one stored. If +so, the user is accepted. Is this password-based authentication scheme secure? + +## Programming in not-C + +When we write programs, we should avoid languages like C or C++. (This is even +suggested by some US government agencies.) It's better to use a language like +Rust, which has built-in memory safety. Writing programs in Rust will result in +fewer security vulnerabilities in the programs and will thus be more secure. Is +this really true? + +## Adding features + +The Bank has used the same back-end software for 50 years (written in COBOL). +This software works and manages the Bank's accounts. Previously the Bank's +staff interacted with the software, e.g. to tranfer money between accounts. + +The Bank hired FutureSoftware to add a web-based front-end so that the +customers can interact with the software directly. That way customers can +transfer money between accounts without the need for Bank staff to do it for +them. (They wrote the software in Python.) + +The Bank then asked VETCYB consultants to evaluate if this is secure. How would +go do it? + +## Mandatory Access Control (MAC) + +The implementation of Mandatory Access Control (MAC) mechanisms, such +as the Bell-LaPadula Model, is considered crucial in environments that +demand high levels of security and confidentiality. Organizations that +handle sensitive or classified information must ensure that data access +is strictly regulated. MAC models achieve this by enforcing control +policies that limit access based on user clearance levels. Does this work? + +## Multi-Factor Authentication (MFA) + +Multi-Factor Authentication (MFA) is promoted as an effective way to enhance +security. By requiring users to provide two or more verification factors before +granting access, MFA adds an additional layer of protection beyond just the +password. The methods of implementing MFA varies; such as SMS, email, or +authenticator apps. Is this secure? + +## Firewalls + +Firewalls are a fundamental component of network security that are +taught in introductory courses. They act as barriers between trusted and +untrusted networks by controlling incoming and outgoing network traffic +based on predetermined security rules. Firewalls can prevent +unauthorized access and attacks. Is this true? + +## Social Engineering Awareness Training + +Security courses often emphasize the importance of educating employees +about social engineering attacks. The training aims to help employees +recognize and avoid common tactics used in attacks, such as phishing. +Is this secure? How effective, if at all, is such training in practice at +preventing social engineering breaches over time? + +## Encrypt Data-at-Rest + +One common security practice is encrypting data-at-rest to protect +sensitive information stored on devices or servers. By encrypting the +data, unauthorized users are prevented from accessing it without the +appropriate decryption key. Is this secure? + +## Regular Security Updates + +It is standard advice to regularly update software to stay protected +against newly discovered vulnerabilities. Updates often include patches +for security flaws that could be exploited by attackers. Does this contribute +to increased security? + +## Penetration Testing (Pentesting) + +Penetration testing, commonly referred to as pentesting, is a proactive +approach to evaluating the security of a computer system, network, +or application by simulating an attack from malicious outsiders and +insiders. Pentesting can help identify vulnerabilities before attackers +exploit them. A comprehensive pentest involves several stages, including +planning, reconnaissance, scanning, exploitation, and reporting. After a +successful pentest the system will be secure. Is this true? diff --git a/modules/05-scientificity/seminar-slides.tex b/modules/05-scientificity/seminar-slides.tex new file mode 100644 index 0000000..3131b46 --- /dev/null +++ b/modules/05-scientificity/seminar-slides.tex @@ -0,0 +1,12 @@ +\documentclass[ignoreframetext]{beamer} +\input{preamble.tex} + +\begin{document} +\mode +\input{seminar.tex} +\mode* + +\begin{frame}[allowframebreaks] + \printbibliography +\end{frame} +\end{document} diff --git a/modules/05-scientificity/seminar.md b/modules/05-scientificity/seminar.md new file mode 100644 index 0000000..214c67f --- /dev/null +++ b/modules/05-scientificity/seminar.md @@ -0,0 +1,11 @@ +We will talk about scientificity in security. + +**Participation**: online in the class Zoom room. + +**Preparation**: The material in this module that precedes this page. + +**Seminar content**: Briefly discuss the methods and research questions of the +papers. Discuss the role of the scientific method in security and what doing +security in a scientific way means. + +We'll also introduce the final assignment. diff --git a/modules/05-scientificity/seminar.tex b/modules/05-scientificity/seminar.tex new file mode 100644 index 0000000..b563edc --- /dev/null +++ b/modules/05-scientificity/seminar.tex @@ -0,0 +1,319 @@ +\title{% + Science in Security, + wrapping up +} +\author{Daniel Bosk\thanks{% + This material is available under the Creative Commons + Attribution-NonCommercial-ShareAlike (CC-BY-NC-SA) 4.0 international + license. + The material was written with some aid from GitHub Copilot. +}} +\institute{% + KTH EECS +} + +\begin{frame} + \maketitle +\end{frame} + +\mode* + +\begin{abstract} + \input{abstract.tex} +\end{abstract} + +\clearpage + +\section{Security as a scientific pursuit} + +\Textcite{SoKScienceOfSecurity} discusses security as a scientific pursuit. + +\begin{frame} + \begin{exercise}[Time to vent] + Air your thoughts about the paper + \citetitle{SoKScienceOfSecurity}~\autocite{SoKScienceOfSecurity}. + \end{exercise} + + \begin{center} + \(<10\) minutes + \end{center} +\end{frame} + +\begin{frame} + \begin{question} + How did they\autocite{SoKScienceOfSecurity} know that this is so? + What can we actually say from these results? + \end{question} + + \begin{question} + The paper\autocite{SoKScienceOfSecurity} is from + \citeyear{SoKScienceOfSecurity}. + How well can we repeat this study to see what the state is today? + \end{question} + + \begin{center} + \(<10\) minutes + \end{center} +\end{frame} + +\begin{frame} + \begin{question} + What can empirical experiments contribute to security? + \end{question} + + \begin{question} + What can deductive reasoning contribute to security? + \end{question} + + \begin{question} + What can quantitative observations contribute to security? + \end{question} + + \begin{question} + What can qualitative observations contribute to security? + \end{question} + + \begin{center} + 5 minutes + \end{center} +\end{frame} + +\begin{frame} + \begin{remark}[Final take-away] + Always look for \enquote{the full version of the paper}! + \end{remark} +\end{frame} + + +\section{Some cases} + +\begin{frame}[fragile] + \begin{block}{The cases} + \begin{itemize} + \item Passwords + \item Programming in not-C + \item Adding features + \item Mandatory Access Control (MAC) + \item Multi-Factor Authentication (MFA) + \item Firewalls + \item Social Engineering Awareness Training + \item Encrypt Data-at-Rest + \item Regular Security Updates + \item Penetration Testing (Pentesting) + \end{itemize} + \end{block} +\end{frame} + +\begin{frame}[fragile] + \begin{exercise}[Work out cases] + \begin{itemize} + \item Join the breakout room of your favourite case. + \item Work out the \emph{methodology} for \emph{how to know}. + \hfill 15 minutes% erase the space, otherwise these will be uneven + \begin{itemize} + \item Analyse and evaluate your different approaches. + \item Try to settle on one. + \end{itemize} + \item Take a break. + \hfill 10 minutes% + \item Then you'll summarize in whole class. + \hfill 50 minutes% + \end{itemize} + \end{exercise} + + \begin{question}[To answer in class] + \begin{itemize} + \item What did you conclude: how do you \emph{evaluate} if a solution + is correct? + \item What were the disagreements? + \end{itemize} + \end{question} +\end{frame} + +\begin{frame} + \begin{block}{Passwords} + To do password-based authentication, we're taught that we should salt + and hash the passwords. The user submits the password to the server, + the server adds a 128-bit salt (random number) and hashes the password + using SHA256, then hashes the hash, and hashes that hash, and so on for 1000 + times. That final hash is used to check if it's the same as the one stored. If + so, the user is accepted. Is this password-based authentication scheme secure? + \end{block} + + \begin{question} + \begin{itemize} + \item What did you conclude: how do you \emph{evaluate} if this is + correct? + \item What were the disagreements? + \end{itemize} + \end{question} +\end{frame} + +\begin{frame} + \begin{block}{Programming in not-C} + When we write programs, we should avoid languages like C or C++. (This is even + suggested by some US government agencies.) It's better to use a language like + Rust, which has built-in memory safety. Writing programs in Rust will result in + fewer security vulnerabilities in the programs and will thus be more secure. Is + this really true? + \end{block} + + \begin{question} + \begin{itemize} + \item What did you conclude: how do you \emph{evaluate} if this is + correct? + \item What were the disagreements? + \end{itemize} + \end{question} +\end{frame} + +\begin{frame} + \begin{block}{Adding features} + The Bank has used the same back-end software for 50 years (written in COBOL). + This software works and manages the Bank's accounts. Previously the Bank's + staff interacted with the software, e.g. to tranfer money between accounts. + + The Bank hired FutureSoftware to add a web-based front-end so that the + customers can interact with the software directly. That way customers can + transfer money between accounts without the need for Bank staff to do it for + them. (They wrote the software in Python.) + + The Bank then asked VETCYB consultants to evaluate if this is secure. How would + go do it? + \end{block} + + \begin{question} + \begin{itemize} + \item What did you conclude: how do you \emph{evaluate} if this is + correct? + \item What were the disagreements? + \end{itemize} + \end{question} +\end{frame} + +\begin{frame} + \begin{block}{Mandatory Access Control (MAC)} + The implementation of Mandatory Access Control (MAC) mechanisms, such + as the Bell-LaPadula Model, is considered crucial in environments that + demand high levels of security and confidentiality. Organizations that + handle sensitive or classified information must ensure that data access + is strictly regulated. MAC models achieve this by enforcing control + policies that limit access based on user clearance levels. Does this work? + \end{block} + + \begin{question} + \begin{itemize} + \item What did you conclude: how do you \emph{evaluate} if this is + correct? + \item What were the disagreements? + \end{itemize} + \end{question} +\end{frame} + +\begin{frame} + \begin{block}{Multi-Factor Authentication (MFA)} + Multi-Factor Authentication (MFA) is promoted as an effective way to enhance + security. By requiring users to provide two or more verification factors before + granting access, MFA adds an additional layer of protection beyond just the + password. The methods of implementing MFA varies; such as SMS, email, or + authenticator apps. Is this secure? + \end{block} + + \begin{question} + \begin{itemize} + \item What did you conclude: how do you \emph{evaluate} if this is + correct? + \item What were the disagreements? + \end{itemize} + \end{question} +\end{frame} + +\begin{frame} + \begin{block}{Firewalls} + Firewalls are a fundamental component of network security that are + taught in introductory courses. They act as barriers between trusted and + untrusted networks by controlling incoming and outgoing network traffic + based on predetermined security rules. Firewalls can prevent + unauthorized access and attacks. Is this true? + \end{block} + + \begin{question} + \begin{itemize} + \item What did you conclude: how do you \emph{evaluate} if this is + correct? + \item What were the disagreements? + \end{itemize} + \end{question} +\end{frame} + +\begin{frame} + \begin{block}{Social Engineering Awareness Training} + Security courses often emphasize the importance of educating employees + about social engineering attacks. The training aims to help employees + recognize and avoid common tactics used in attacks, such as phishing. + Is this secure? How effective, if at all, is such training in practice at + preventing social engineering breaches over time? + \end{block} + + \begin{question} + \begin{itemize} + \item What did you conclude: how do you \emph{evaluate} if this is + correct? + \item What were the disagreements? + \end{itemize} + \end{question} +\end{frame} + +\begin{frame} + \begin{block}{Encrypt Data-at-Rest} + One common security practice is encrypting data-at-rest to protect + sensitive information stored on devices or servers. By encrypting the + data, unauthorized users are prevented from accessing it without the + appropriate decryption key. Is this secure? + \end{block} + + \begin{question} + \begin{itemize} + \item What did you conclude: how do you \emph{evaluate} if this is + correct? + \item What were the disagreements? + \end{itemize} + \end{question} +\end{frame} + +\begin{frame} + \begin{block}{Regular Security Updates} + It is standard advice to regularly update software to stay protected + against newly discovered vulnerabilities. Updates often include patches + for security flaws that could be exploited by attackers. Does this contribute + to increased security? + \end{block} + + \begin{question} + \begin{itemize} + \item What did you conclude: how do you \emph{evaluate} if this is + correct? + \item What were the disagreements? + \end{itemize} + \end{question} +\end{frame} + +\begin{frame} + \begin{block}{Penetration Testing (Pentesting)} + Penetration testing, commonly referred to as pentesting, is a proactive + approach to evaluating the security of a computer system, network, + or application by simulating an attack from malicious outsiders and + insiders. Pentesting can help identify vulnerabilities before attackers + exploit them. A comprehensive pentest involves several stages, including + planning, reconnaissance, scanning, exploitation, and reporting. After a + successful pentest the system will be secure. Is this true? + \end{block} + + \begin{question} + \begin{itemize} + \item What did you conclude: how do you \emph{evaluate} if this is + correct? + \item What were the disagreements? + \end{itemize} + \end{question} +\end{frame} diff --git a/modules/05-scientificity/slides.tex b/modules/05-scientificity/slides.tex new file mode 100644 index 0000000..a77bbb7 --- /dev/null +++ b/modules/05-scientificity/slides.tex @@ -0,0 +1,12 @@ +\documentclass[ignoreframetext]{beamer} +\input{preamble.tex} + +\begin{document} +\mode +\input{contents.tex} +\mode* + +\begin{frame}[allowframebreaks] + \printbibliography +\end{frame} +\end{document} diff --git a/modules/Makefile b/modules/Makefile index 6ce53d3..2737eef 100644 --- a/modules/Makefile +++ b/modules/Makefile @@ -1,3 +1,4 @@ +SUBDIR+= ./00-intro SUBDIR+= ./01-methods-overview SUBDIR+= ./02-passwd diff --git a/modules/final-seminar/Makefile b/modules/final-seminar/Makefile index 553f2f0..ebdbaf2 100644 --- a/modules/final-seminar/Makefile +++ b/modules/final-seminar/Makefile @@ -14,7 +14,7 @@ notes.pdf: ${SRC} ${EXT_SRC} slides.pdf: slides.tex slides.pdf: ${SRC} ${EXT_SRC} -objectives.tex: ../../course-design/objectives.tex +objectives.tex: ../00-intro/objectives.tex ${EXT_SRC}: ${LN} $< $@ diff --git a/modules/final-seminar/contents.tex b/modules/final-seminar/contents.tex index 4307019..84748f2 100644 --- a/modules/final-seminar/contents.tex +++ b/modules/final-seminar/contents.tex @@ -36,6 +36,8 @@ \section{Introduction} \begin{frame} \begin{block}{Goal} \begin{itemize} + \item You should be able to evaluate a proposed method to answer a + question. \item You should be able to design a method to answer a research question. \item Security is multifaceted, you should be able to consider more than @@ -60,16 +62,12 @@ \section{Scenario and research question} The research question that you should answer is the following: \begin{frame} -\begin{question} - We want to develop a secure instant-messaging system\footnote{% - Think chat or SMS/text messages; something like Signal, WhatsApp, Telegram, - \etc - }. - How can we evaluate its security? -\end{question} +\begin{restatable}{question}{mainRQ}\label{mainRQ} + How secure is the Signal instant-messaging app? +\end{restatable} \end{frame} -\section{Assessment}\label{Assessment} +\section
{Assessment}\label{Assessment} In brief, what you should be able to do is to ask good questions and propose suitable ways of answering them. @@ -87,42 +85,73 @@ \subsection{Material you should produce} To be able to show that you can do that, you should \begin{frame} - \only{You should} \begin{itemize} \item write a report, and - \item make a presentation (10 minutes).\footnote{% - If you write your report in LaTeX, consider writing your slides in LaTeX - also using the \texttt{beamer} and \texttt{beamerarticle} packages. - } + \item review someone else's report. \end{itemize} -These should should contain the following sections: +\end{frame} + +The report should contain the following sections: +\begin{frame} \begin{enumerate} - \item Research question overview. - \only
{This section should give an overview of the research - question and outline any subquestions that you derive. - (This focuses on \cref{LOrelate}.)} - \item Methodology. - \only
{This section contains how you propose to answer the - questions from the previous section (methods used). - (This focuses on \cref{LOapply}.)} - \item Discussion. - \only
{In this section you discuss why those methods answer the - questions properly and any limitations that you see. - You can also discuss item alternative methods that you discarded (and - why). - (This focuses on \cref{LOevaluate}.)} + \item Related works. + \only
{% + This section should give an overview of the existing research literature + related to \cref{mainRQ}. + That is, you should find all papers that answers some aspect of it. + You should note what question each paper answers and summarize how they + do that. + (This focuses on \cref{LOevaluate}.)% + }% + \only{% + \begin{itemize} + \item Find all papers answering + some aspect. + \item What (sub)question does each paper answer and how? + \end{itemize} + } + \item Missing aspects. + \only
{% + In this section, you pose questions that are also related to + \cref{mainRQ}, but has not been covered by the literature. + For each such question, you must propose a method that correctly provides + an answer\footnote{% + But remember, you don't have to actually perform any of these + methods. + }. + You must also discuss why this method is suitable to answer the question + and any limitations. + (This focuses on \cref{LOapply,LOevaluate}.)% + }% + \only{% + \begin{itemize} + \item State relevant (sub)questions not asked by the literature. + Propose a method to answer each. + \item \alert<+>{Discuss why the method is suitable and any + limitations.} + \end{itemize} + } \item Conclusion. - \only
{This section ties the sack. - Here you connect the questions and the types of answers gained (through the - methods) and piece them back into the original research question. - You also summarize how well you find the original question to be answered, - if there are any \enquote{holes that need filling}. - (This focuses on \cref{LOrelate,LOevaluate,LOcomm}.)} + \only
{% + This section ties the sack. + Here you connect the questions (yours and from related works) and the + types of answers gained (through the methods) and piece them back into + \cref{mainRQ}. + You also summarize how well you find \cref{mainRQ} to be answered, if + there are any \enquote{holes that need filling}. + (This focuses on \cref{LOrelate}.)% + }% + \only{% + \begin{itemize} + \item Connect the questions and answers back to \cref{mainRQ}. + \item \alert<+>{Discuss how well the main RQ is answered.} + \end{itemize} + } % \item your original plan for the course work, adaptations made and what you % learned. \end{enumerate} \end{frame} -This assesses \cref{LOrelate,LOevaluate,LOapply,LOcomm,LOplan}. +This assesses \cref{LOrelate,LOevaluate,LOapply,LOcomm}. %\paragraph{Feedback you should provide} % @@ -145,7 +174,7 @@ \subsection{Material you should produce} \subsection{Assessment criteria} To assess the learning objectives -(\cref{LOrelate,LOevaluate,LOapply,LOplan,LOcomm}) +(\cref{LOrelate,LOevaluate,LOapply,LOcomm,LOplan}) we use the following criteria. These criteria are also included as a rubric in the assignment where you hand in your report. @@ -156,82 +185,87 @@ \subsection{Assessment criteria} You'll need a pass on all criteria to pass the assignment and the course. -\begin{frame}[fragile,allowframebreaks] - \RaggedRight - \begin{longtable} - {p{0.33\textwidth}p{0.33\textwidth}p{0.33\textwidth}p{0.33\textwidth}} - \toprule - \textbf{Learning objective} - & \textbf{Criteria} - & \textbf{Pass} - & \textbf{Fail} - \\* - \midrule - \endhead - \only
{\cref{LOrelate}:} - The student is able to \LOrelate - & The main research question is explored from relevant aspects? - & There might be more aspects to explore, but the most important ones are - covered. Motivate why no more aspect need to be explored. - & There is at least one aspect missing that can be motivated to be - important. Motivate which one. - For instance, do we need to ask another more detailed (research) question - to be able to answer the main research question in a meaningful way? Do - they address the question from just a single perspective? - \\* - \newpage - \only
{\cref{LOapply}:} - The student is able to \LOapply - & The methods are suitable to answer the questions? - & All questions have suggested methods that can actually answer the - question correctly. - Motivate why this is the case. - & There is at least one question that will not be answered correctly with - the suggested method. - State which one and why. - For instance, the method might only answer part of the question. - %Or not at all. - \\* - \newpage - \only
{\cref{LOevaluate}:} - The student is able to \LOevaluate - & Are all quality aspects considered in the discussion? - & The most important quality aspects are considered and discussed. - & At least one important quality aspect is missing. - State which one and motivate why it's important enough that it must be - treated. - \\* - & Are all ethical aspects considered? - & The most important ethical aspects are considered and discussed. - & At least one important ethical aspect is missing. - State which one and motivate why it's important enough that it must be - treated. - \\* - \newpage - \only
{\cref{LOcomm}:} - The student is able to \LOcomm - & Is the report written as short as possible? - & The report can probably be slightly shortened, but not by much. - & The report can be shortened considerably. - Give at least one example of where and how. - \\* - & Is the report clear and easy to understand? - & The report is easy to understand. - & Some parts of the report must be read more than once to understand. - (Or worse.) - Give at least one example. - \\* - & Are the arguments clearly stated and well motivated? - & All arguments are clearly stated and well motivated. - & At least one argument is not clearly stated or not well motivated. - State which one and motivate why it's not clear or well motivated. - \\* - \bottomrule - \end{longtable} +\begin{frame} + \begin{block}{Assessment criteria} + \begin{itemize} + \item \alert<+>\LOrelate + \item \LOapply + \item \alert<+>\LOevaluate + \item \LOcomm + \end{itemize} + \end{block} \end{frame} +{\RaggedRight +\begin{fullwidth} +\begin{longtable} +{p{0.20\columnwidth}p{0.20\columnwidth}p{0.20\columnwidth}p{0.20\columnwidth}} +\toprule +\textbf{Learning objective} + & \textbf{Criteria} + & \textbf{Pass} + & \textbf{Fail} + \\* +\midrule +\endhead +\only
{\cref{LOrelate}:} +The student is able to \LOrelate + & The main research question is explored from relevant aspects? + & There might be more aspects to explore, but the most important ones are + covered. Motivate why no more aspect need to be explored. + & There is at least one aspect missing that can be motivated to be + important. Motivate which one. + For instance, do we need to ask another more detailed (research) question + to be able to answer the main research question in a meaningful way? Do + they address the question from just a single perspective? + \\* +\only
{\cref{LOapply}:} +The student is able to \LOapply + & The methods are suitable to answer the questions? + & All questions have suggested methods that can actually answer the + question correctly. + Motivate why this is the case. + & There is at least one question that will not be answered correctly with + the suggested method. + State which one and why. + For instance, the method might only answer part of the question. + %Or not at all. + \\* +\only
{\cref{LOevaluate}:} +The student is able to \LOevaluate + & Are all quality aspects considered in the discussion? + & The most important quality aspects are considered and discussed. + & At least one important quality aspect is missing. + State which one and motivate why it's important enough that it must be + treated. + \\* + & Are all ethical aspects considered? + & The most important ethical aspects are considered and discussed. + & At least one important ethical aspect is missing. + State which one and motivate why it's important enough that it must be + treated. + \\* +\only
{\cref{LOcomm}:} +The student is able to \LOcomm + & Is the report written as short as possible? + & The report can probably be slightly shortened, but not by much. + & The report can be shortened considerably. + Give at least one example of where and how. + \\* + & Is the report clear and easy to understand? + & The report is easy to understand. + & Some parts of the report must be read more than once to understand. + (Or worse.) + Give at least one example. + \\* + & Are the arguments clearly stated and well motivated? + & All arguments are clearly stated and well motivated. + & At least one argument is not clearly stated or not well motivated. + State which one and motivate why it's not clear or well motivated. +\end{longtable} +\end{fullwidth} +} -\only
{% -\subsection{Plagiarism} +\subsection
{Plagiarism} You work in the groups that you've signed up for. You may discuss with others, search the literature and use tools such as @@ -254,33 +288,55 @@ \subsection{Plagiarism} Not mentioning that you've used ChatGPT or discussed the topic with people outside the group is considered an attempt to deceive. -} \section{The final seminar} -\subsection{Structure} +During the final seminar we will discuss the questions and methods that you've +covered in the report and how well \cref{mainRQ} was covered. -\begin{frame} - \begin{block}{Structure} - \begin{enumerate} - \item Each group presents - \item After each presentation, I might ask for some more details. - \item When everyone has presented, we discuss what we've heard. - \end{enumerate} - \end{block} -\end{frame} +For the seminar you should prepare slides. +Each slide should focus on a research question. +If the research question was covered by the literature, you should have a +reference to the paper on the slide. -\subsection{Questions for discussion} +During the seminar, with the slide you should be able to explain the question +and your evaluation of how well the method answers the question. +(You can use more than one slide if it improves the presentation, but it +shouldn't be necessary.) -\begin{frame} - \begin{question} - What are the first thoughts while/after hearing the others present? - \end{question} - \begin{question} - What did you like the most from what you've heard? - \end{question} - \begin{question} - What would you like to improve/do differently/add to your own work - after hearing what the others did? - \end{question} +The participants will take turn in presenting the question, method and +evaluation. +The we discuss jointly. +We will want to discuss closely related questions, try to group them by that in +the slide deck. + +\begin{frame} + \begin{activity}[Order] + \begin{itemize} + \item What order to present? + \item You'll have one minute to write \enquote{me!}. + \item The last one to write \enquote{me!} goes first. + \end{itemize} + \end{activity} +\end{frame} + +\begin{frame} + \mainRQ* + \begin{activity} + \begin{itemize} + \item The presenter's favourite aspect/subquestion first. + \item Summarize the question and the method\footnote{% + Cite if from a paper. + }. + \item Discuss why you think this is good\footnote{% + Focuses on the weak points pointed out above. + }: + \begin{itemize} + \item Why does this method answer the question? + \item Why does the answer to the question help answering the main RQ. + \item What ethical aspects are there to consider? + \end{itemize} + \item What do the others think? + \end{itemize} + \end{activity} \end{frame} diff --git a/modules/final-seminar/notes.tex b/modules/final-seminar/notes.tex index 30c6191..c8f6dae 100644 --- a/modules/final-seminar/notes.tex +++ b/modules/final-seminar/notes.tex @@ -1,39 +1,4 @@ \documentclass[a4paper,10pt,article,oneside]{memoir} -%%% Tufte %%% -\usepackage{marginfix} -%\setlength{\evensidemargin}{\oddsidemargin} -\marginparmargin{outer} -\setlrmarginsandblock{2.5cm}{8cm}{*} - -\footnotesinmargin - -\usepackage{ragged2e} -\renewcommand{\sidefootform}{\RaggedRight} -\renewcommand{\foottextfont}{\footnotesize\RaggedRight} - -\setmpjustification{\RaggedRight}{\RaggedRight} - -% margin figure and caption typeset ragged against text block -\setfloatadjustment{marginfigure}{\mpjustification} -\setmarginfloatcaptionadjustment{figure}{\captionstyle{\mpjustification}} - -% From https://tex.stackexchange.com/a/324757/17418 -% Palatino for main text and math -\usepackage[osf,sc]{mathpazo} - -% Helvetica for sans serif -% (scaled to match size of Palatino) -\usepackage[scaled=0.90]{helvet} - -% Bera Mono for monospaced -% (scaled to match size of Palatino) -\usepackage[scaled=0.85]{beramono} - -\setlxvchars\setxlvchars -\checkandfixthelayout - -\nouppercaseheads -%%% end tufte %%% \let\subsubsection\subsection \let\subsection\section \let\section\chapter @@ -46,17 +11,6 @@ %\usepackage{authblk} %\let\institute\affil -\declaretheorem[style=theorem]{theorem} -\declaretheorem[style=definition]{definition} -\declaretheorem[style=definition]{assumption} -\declaretheorem[style=definition]{protocol} -\declaretheorem[style=example]{example} -\declaretheorem[style=remark]{remark} -\declaretheorem[style=remark]{idea} -\declaretheorem[style=exercise]{exercise} -\declaretheorem[style=exercise]{question} -\declaretheorem[style=solution]{solution} - \usepackage[inline]{enumitem} \begin{document} diff --git a/modules/final-seminar/preamble.tex b/modules/final-seminar/preamble.tex index 0789d3b..f6f9822 100644 --- a/modules/final-seminar/preamble.tex +++ b/modules/final-seminar/preamble.tex @@ -41,6 +41,8 @@ \usepackage[binary-units]{siunitx} +\usepackage{didactic} + \usepackage[capitalize]{cleveref} \input{objectives.tex} diff --git a/modules/final-seminar/slides.tex b/modules/final-seminar/slides.tex index 2287028..a77bbb7 100644 --- a/modules/final-seminar/slides.tex +++ b/modules/final-seminar/slides.tex @@ -1,100 +1,6 @@ \documentclass[ignoreframetext]{beamer} \input{preamble.tex} -\usetheme{Berlin} -\setbeamertemplate{footline}%{miniframes theme} -{% - \begin{beamercolorbox}[colsep=1.5pt]{upper separation line foot} - \end{beamercolorbox} - \begin{beamercolorbox}[ht=2.5ex,dp=1.125ex,% - leftskip=.3cm,rightskip=.3cm plus1fil]{author in head/foot}% - \leavevmode{\usebeamerfont{author in head/foot}\insertshortauthor}% - \hfill% - {\usebeamerfont{institute in head/foot}\usebeamercolor[fg]{institute in head/foot}\insertshortinstitute}% - \end{beamercolorbox}% - \begin{beamercolorbox}[ht=2.5ex,dp=1.125ex,% - leftskip=.3cm,rightskip=.3cm plus1fil]{title in head/foot}% - {\usebeamerfont{title in head/foot}\insertshorttitle} \hfill \insertframenumber% - \end{beamercolorbox}% - \begin{beamercolorbox}[colsep=1.5pt]{lower separation line foot} - \end{beamercolorbox} -} -\setbeamercovered{transparent} -\setbeamertemplate{bibliography item}[text] - -\AtBeginSection[]{% - \begin{frame} - \tableofcontents[currentsection] - \end{frame} -} - -\ProvideDocumentEnvironment{assumption}{o}{% - \IfValueTF{#1}{% - \begin{block}{Assumption: #1} - }{% - \begin{block}{Assumption} - } -}{% - \end{block} -} - -\ProvideDocumentEnvironment{protocol}{o}{% - \IfValueTF{#1}{% - \begin{block}{Protocol: #1} - }{% - \begin{block}{Protocol} - } -}{% - \end{block} -} - -\ProvideDocumentEnvironment{remark}{o}{% - \IfValueTF{#1}{% - \begin{alertblock}{Note: #1} - }{% - \begin{alertblock}{Note} - } -}{% - \end{alertblock} -} - -\ProvideDocumentEnvironment{idea}{o}{% - \IfValueTF{#1}{% - \begin{block}{Idea: #1} - }{% - \begin{block}{Idea} - } -}{% - \end{block} -} - -\ProvideDocumentEnvironment{question}{o}{% - \setbeamercolor{block body}{bg=orange!15,fg=black} - \setbeamercolor{block title}{bg=orange,fg=white} - \setbeamercolor{local structure}{fg=orange} - \IfValueTF{#1}{% - \begin{block}{Question: #1} - }{% - \begin{block}{Question} - } -}{% - \end{block} -} - -\ProvideDocumentEnvironment{exercise}{o}{% - \setbeamercolor{block body}{bg=yellow!10,fg=black} - \setbeamercolor{block title}{bg=yellow,fg=black} - \setbeamercolor{local structure}{fg=yellow} - \IfValueTF{#1}{% - \begin{block}{Exercise: #1} - }{% - \begin{block}{Exercise} - } -}{% - \end{block} -} - - \begin{document} \mode \input{contents.tex}