The first limit you'll hit on AWS EFS: Locks

df -h
fs-<...>.amazonaws.com:/  8.0E  1.6G  8.0E   1% /mnt/nfs

# Create a container named `inst1` which has
# two environment variables set which alters
# the behavior of the mysql instance startup.
#
# The code that runs at startup time can be found
# at the repository that holds this image:
# https://hub.docker.com/mysql
docker run \
        --detach \
        --name inst1 \
        --env MYSQL_DATABASE=testdb \
        --env MYSQL_ALLOW_EMPTY_PASSWORD=true \
        mysql

# If the string `MYSQL_DATABASE` has been
# specified, then add the database creation
# string to the list of mysql commands to 
# run after the process has started.
# (line 159 of docker-entrypoint.sh)
if [ "$MYSQL_DATABASE" ]; then
        echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` ;" | \
                "${mysql[@]}"
        mysql+=( "$MYSQL_DATABASE" )
fi

# get into the container
docker exec \
        --interactive \
        --tty \
        inst1 \
        /bin/bash


# From within the container, get into the directory 
# where mysql puts our database files and everything 
# that we should persist
cd /var/lib/mysql


# Find all the directories that are out there
find . -type d

        .
        ./sys
        ./mysql
        ./testdb
        ./performance_schema


# Check that indeed, these are just databases
echo "SHOW DATABASES;" | mysql 

        Database
        information_schema
        mysql                   <<<<<<<
        performance_schema      <<<<<<<
        sys
        testdb                  <<<<<<<

lslocks
COMMAND PID  TYPE SIZE MODE  M START END PATH
mysqld    1 POSIX  48M WRITE 0     0   0 ./ib_logfile0
mysqld    1 POSIX  48M WRITE 0     0   0 ./ib_logfile1
mysqld    1 POSIX  76M WRITE 0     0   0 ./ibdata1
mysqld    1 POSIX  12M WRITE 0     0   0 ./ibtmp1
mysqld    1 POSIX  96K WRITE 0     0   0 ./mysql/plugin.ibd
mysqld    1 POSIX  96K WRITE 0     0   0 ./mysql/innodb_table_stats.ibd
mysqld    1 POSIX  96K WRITE 0     0   0 ./mysql/innodb_index_stats.ibd
mysqld    1 POSIX  96K WRITE 0     0   0 ./mysql/gtid_executed.ibd
mysqld    1 POSIX  96K WRITE 0     0   0 ./mysql/server_cost.ibd
mysqld    1 POSIX  96K WRITE 0     0   0 ./mysql/engine_cost.ibd
mysqld    1 POSIX  96K WRITE 0     0   0 ./mysql/time_zone_leap_second.ibd
mysqld    1 POSIX  96K WRITE 0     0   0 ./mysql/servers.ibd
mysqld    1 POSIX 320K WRITE 0     0   0 ./mysql/time_zone_name.ibd
mysqld    1 POSIX 144K WRITE 0     0   0 ./mysql/time_zone.ibd
mysqld    1 POSIX  12M WRITE 0     0   0 ./mysql/time_zone_transition.ibd
mysqld    1 POSIX 464K WRITE 0     0   0 ./mysql/time_zone_transition_type.ibd

# Create the query that creates a table called 
# `example` with just two dummy columns.
table_creation_query="
CREATE TABLE example ( 
        id INT, 
        data VARCHAR(100) 
);"


# Execute the query against our database
mysql --database testdb <<< "$table_creation_query"


# Verify that a lock has been created
lslocks | grep testdb
mysqld    1 POSIX  96K WRITE 0     0   0 ./testdb/example.ibd

# Get the PID of the MySQL daemon instance as seen by
# the host.
docker top inst1
PID          ..   CMD
861379       ..   mysqld
861589       ..   /bin/bash
864033       ..   /bin/bash


# Cool, PID of `mysqld` is 861379.
# From the host we can specify that pid to strace
# and tell it to attach to all the processes raised 
# by it. 
#
# This way we can look at every syscall
# made by them while it runs.
#
# Here I'm making use of the following arguments:
# -f:   Trace  child  processes (and "lightweight processes")
# -e:   Filter the calls so that we only get notified when
#       there's a syscall that matches the term
# -p:   Pid to attach to (and its children)

sudo strace -f -e fcntl -p 861379 
strace: Process 861379 attached with 28 threads


# Now that we got attached, in a separate terminal, create a new
# table by sending the table creation command to the
# container's mysql binary (client) that will get the command
# sent to the mysql daemon.
echo "CREATE TABLE example2 ( id INT, data VARCHAR(100) );" | 
        docker exec inst1 \
                mysql --database testdb


# Now look at our strace results!
# We spotted some `fcntl` syscalls with
# the lock-related parameters
[pid 862376] fcntl(44, F_SETLK, {       <<<<<
        l_type=F_WRLCK,                 <<<<<
        l_whence=SEEK_SET, 
        l_start=0, 
        l_len=0}) = 0

[pid 862376] fcntl(44, F_SETLK, {       <<<<<<
        l_type=F_WRLCK,                 <<<<<<
        l_whence=SEEK_SET, 
        l_start=0, 
        l_len=0}) = 0

# Get inside the container
docker exec -it inst1 /bin/bash

# By looking at the proc virtual filesystem we
# can check what files are open.
#
# Because we're running a container that runs MYSQL
# as the PID 1, we look at the properties for the 
# process who's PID is 1.
#
readlink /proc/1/fd/44 
/var/lib/mysql/testdb/example2.ibd

# Run the same command as before that spawns
# a mysql container but this time with the limit
# of 25 locks. 
docker run \
        --detach \
        --name inst1 \
        --ulimit locks=25 \
        --env MYSQL_DATABASE=testdb \
        --env MYSQL_ALLOW_EMPTY_PASSWORD=true \
        mysql

#!/bin/bash

# exit on any failed execution
set -o errexit

# main routine that calls the
# table creation method 30 times
main () {
        for i in $(seq 1 30); do
                create_table mytable$i
        done
}

# execute a command within the mysql
# container to create a table.
# args:
#       1) name of the table
create_table () {
        local name=$1

        echo "CREATE TABLE $name ( 
        id INT, 
        data VARCHAR(100) 
);" | docker exec -i inst1 \
        mysql --database=testdb
}

main

# Run `strace` targetting the process at pid $PID
# as well as it's children.
#
# Inject an FCNTL error in case the syscall has been
# executes more than 25 times.
strace \
        -p $PID \
        -f \
        -e fault=fcntl:error=ENOLCK:when=25+

.
├── main.tf             # data retrieval and output
├── efs.tf              # provisiong the filesystem and a mount target
│                       # in the specific AZ that our instance is in
├── instance.tf         # creates an instance in the region and AZ we want
├── keys.tf             # provisions the key in the region
├── keys                # dumb keys for this one-off experiment
│   ├── key.rsa
│   └── key.rsa.pub
└── terraform.tfstate   # state

1 directory, 7 files

# Creates a new empty file system in EFS.
resource "aws_efs_file_system" "main" {
  tags {
    Name = "locks-test-fs"
  }
}

# Creates a mount target of EFS in a specified subnet
# such that our instances can connect to it.
resource "aws_efs_mount_target" "main" {
  file_system_id  = "${aws_efs_file_system.main.id}"
  subnet_id       = "${var.subnet}"
  security_groups = ["${aws_security_group.efs.id}"]
}

# Allows both ingress and egress for port 2049 (nfs)
# such that our instances can get to the mount
# target in the AZ.
resource "aws_security_group" "efs" {
  name        = "efs-mnt"
  description = "Allows NFS traffic from instances within the VPC."
  vpc_id      = "${data.aws_vpc.default.id}"

  ingress {
    from_port = 2049
    to_port   = 2049
    protocol  = "tcp"

    cidr_blocks = [
      "${data.aws_vpc.default.cidr_block}",
    ]
  }

  egress {
    from_port = 2049
    to_port   = 2049
    protocol  = "tcp"

    cidr_blocks = [
      "${data.aws_vpc.default.cidr_block}",
    ]
  }

  tags {
    Name = "allow_nfs-ec2"
  }
}

#!/bin/bash

# Performs the mounting of an EFS mount target into
# predefined MOUNT_LOCATION in the filesystem.


set -o errexit

readonly MOUNT_LOCATION="${MOUNT_LOCATION:-/mnt/efs}"
readonly MOUNT_TARGET="${MOUNT_TARGET}"


# Main routine - checks for the MOUNT_TARGET variable
# that does not have a default (can be empty) making 
# sure that it gets specified.
main () {
        if [[ -z "$MOUNT_TARGET" ]]; then
                echo 'ERROR: 
        MOUNT_TARGET environment variable must be specified.
                '
                exit 1
        fi

        echo "INFO:
        Mounting shared filesystem.

        MOUNT_LOCATION  $MOUNT_LOCATION
        MOUNT_TARGET    $MOUNT_TARGET
        "

        mount_nfs
}


# Performs the actual mounting of the EFS target
# in the AZ we specified into a directory we defined
# via MOUNT_LOCATION.
mount_nfs() {
        sudo mkdir -p $MOUNT_LOCATION

        sudo mount \
                -t nfs4 \
                -o nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2 \
                $MOUNT_TARGET:/ $MOUNT_LOCATION
}

main

local number_of_tables="$1"

if [[ -z "$number_of_tables" ]]; then
        echo "ERROR:
        An argument (number of tables) must be supplied.
        "
        exit 1
fi

for i in $(seq 1 $number_of_tables); do
        create_table mytable$i
done

./create-many-tables.sh 300

ERROR 1030 (HY000) at line 1: Got error 168 from storage engine

# docker logs ...
[Note] mysqld: ready for connections.
[ERROR] InnoDB: Unable to lock ./testdb/mytable241.ibd error: 122
[ERROR] InnoDB: Operating system error number 122 in a file operation.
[ERROR] InnoDB: Error number 122 means 'Disk quota exceeded'
[ERROR] InnoDB: Cannot create file './testdb/mytable241.ibd'

/**
 * Struct that keeps track of the runtime arguments
 * that we supply via the CLI.
 */ 
typedef struct runtime_t {
	int number_of_locks;
	char* base_directory;
} runtime_t;

/**
 * Prints to STODUT the runtime configuration passed.
 */
void
runtime_show(runtime_t* runtime)
{
	// some printf to show the runtime
}

/**
 * Creates a file under a directory (base) with a number (i)
 * and then grabs a write lock to it.
 */
void
create_file(const char* base, const int i)
{
	FILE* fp;
	char* filename;
	struct flock lock = { 0 };
	int err = 0;

        // make the lock exclusive (write lock)
	lock.l_type = F_WRLCK;

	// .. do some work to have the filename properly set
        // try to acquire the lock
	err = fcntl(fileno(fp), F_SETLKW, &lock);
	if (err == -1) {
		perror("fcntl");
		fprintf(stderr, "Couldn't acquire lock on %s", filename);
		exit(1);
	}

        // just write something stupid to the file
	fprintf(fp, "dummy");
	free(filename);
}

/**
 * Creates `N` files with write a write lock grabbed for each
 * under a base directory `base`.
 */
void
runtime_create_files(runtime_t* runtime)
{
	// ...
}

/**
 * Main execution.
 *
 * Parses the cli arguments (number of locks
 * and base directory) and then starts the creation of the
 * files and locks.
 *
 * Once lock creation is done, pauses its execution until
 * a signal arrives.
 */
int
main(int argc, char** argv)
{
	// ...

	runtime_t runtime = {
		.number_of_locks = atoi(argv[1]), 
                .base_directory = argv[2],
	};

	runtime_show(&runtime);
	runtime_create_files(&runtime);
        
        // ... wait for signals

	return 0;
}