diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DevicePluginScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DevicePluginScheduler.java index 80ffdf7382..479a19db60 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DevicePluginScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/deviceplugin/DevicePluginScheduler.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin; +import java.util.Map; import java.util.Set; /** @@ -29,10 +30,15 @@ public interface DevicePluginScheduler { * Called when allocating devices. The framework will do all device book * keeping and fail recovery. So this hook could be stateless and only do * scheduling based on available devices passed in. It could be - * invoked multiple times by the framework. + * invoked multiple times by the framework. The hint in environment variables + * passed in could be potentially used in making better scheduling decision. + * For instance, GPU scheduling might support different kind of policy. The + * container can set it through environment variables. * @param availableDevices Devices allowed to be chosen from. * @param count Number of device to be allocated. + * @param env Environment variables of the container. * @return A set of {@link Device} allocated * */ - Set allocateDevices(Set availableDevices, int count); + Set allocateDevices(Set availableDevices, int count, + Map env); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nvidia/NvidiaGPUPluginForRuntimeV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nvidia/NvidiaGPUPluginForRuntimeV2.java index 609ec677ca..ee3f54beee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nvidia/NvidiaGPUPluginForRuntimeV2.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nvidia/NvidiaGPUPluginForRuntimeV2.java @@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.Device; import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DevicePlugin; +import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DevicePluginScheduler; import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DeviceRegisterRequest; import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DeviceRuntimeSpec; import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.YarnRuntimeType; @@ -32,7 +33,12 @@ import java.io.File; import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeSet; @@ -40,8 +46,10 @@ /** * Nvidia GPU plugin supporting both Nvidia container runtime v2 for Docker and * non-Docker container. + * It has topology aware as well as simple scheduling ability. * */ -public class NvidiaGPUPluginForRuntimeV2 implements DevicePlugin { +public class NvidiaGPUPluginForRuntimeV2 implements DevicePlugin, + DevicePluginScheduler { public static final Logger LOG = LoggerFactory.getLogger( NvidiaGPUPluginForRuntimeV2.class); @@ -69,6 +77,47 @@ public class NvidiaGPUPluginForRuntimeV2 implements DevicePlugin { private static final Set DEFAULT_BINARY_SEARCH_DIRS = ImmutableSet.of( "/usr/bin", "/bin", "/usr/local/nvidia/bin"); + private boolean topoInitialized = false; + + private Set lastTimeFoundDevices; + + /** + * It caches the combination of different devices and the communication cost. + * The key is device count + * The value is an ordered list of map entry whose key is device combination, + * value is cost. The list is sorted by cost in ascending order. + * For instance: + * { 2=> [[device1,device2]=>0, [device1,device3]=>10] + * 3 => [[device1,device2,device3]=>10, [device2,device3,device5]=>20], + * } + * */ + private Map, Integer>>> costTable + = new HashMap<>(); + + /** + * The key is a pair of minors. For instance, "0-1" indicates 0 to 1 + * The value is weight between the two devices. + * */ + private Map devicePairToWeight = new HashMap<>(); + + /** + * The container can set this environment variable. + * To tell the scheduler what's the policy to use when do scheduling + * */ + public static final String TOPOLOGY_POLICY_ENV_KEY = "NVIDIA_TOPO_POLICY"; + + /** + * Schedule policy that prefer the faster GPU-GPU communication. + * Suitable for heavy GPU computation workload generally. + * */ + public static final String TOPOLOGY_POLICY_PACK = "PACK"; + + /** + * Schedule policy that prefer the faster CPU-GPU communication. + * Suitable for heavy CPU-GPU IO operations generally. + * */ + public static final String TOPOLOGY_POLICY_SPREAD = "SPREAD"; + @Override public DeviceRegisterRequest getRegisterRequestInfo() throws Exception { return DeviceRegisterRequest.Builder.newInstance() @@ -106,6 +155,8 @@ public Set getDevices() throws Exception { id++; } } + // cache it which help to topology scheduling + lastTimeFoundDevices = r; return r; } catch (IOException e) { if (LOG.isDebugEnabled()) { @@ -170,6 +221,422 @@ private String getMajorNumber(String devName) { return output; } + @Override + public Set allocateDevices(Set availableDevices, int count, + Map envs) { + Set allocation = new TreeSet<>(); + /** + * corner cases. + * if allocate 1 device or all devices, no topo scheduling needed. + * if total available devices is less than 3, no topo scheduling needed. + * */ + if (availableDevices.size() < 3 + || count == 1 + || availableDevices.size() == count) { + basicSchedule(allocation, count, availableDevices); + return allocation; + } + + try { + if (!topoInitialized) { + initCostTable(); + } + // topology aware scheduling + topologyAwareSchedule(allocation, count, + envs, availableDevices, this.costTable); + if (allocation.size() == count) { + return allocation; + } else { + LOG.error("Failed to do topology scheduling. Skip to use basic " + + "scheduling"); + } + } catch (IOException e) { + LOG.error("Error in getting GPU topology info. " + + "Skip topology aware scheduling", e); + } + // basic scheduling + basicSchedule(allocation, count, availableDevices); + return allocation; + } + + @VisibleForTesting + public void initCostTable() throws IOException { + // get topology + String topo = shellExecutor.getTopologyInfo(); + // build the graph + parseTopo(topo, devicePairToWeight); + // build the cost table of different device combinations + if (lastTimeFoundDevices == null) { + try { + getDevices(); + } catch (Exception e) { + LOG.error("Failed to get devices!", e); + return; + } + } + buildCostTable(costTable, lastTimeFoundDevices); + loggingCostTable(costTable); + this.topoInitialized = true; + } + + private void loggingCostTable( + Map, Integer>>> cTable) { + if (LOG.isDebugEnabled()) { + StringBuilder sb = new StringBuilder("The costTable is:"); + sb.append("\n{"); + for (Map.Entry, Integer>>> entry + : cTable.entrySet()) { + sb.append("\n\t") + .append(entry.getKey()) + .append(" => ["); + for (Map.Entry, Integer> e : entry.getValue()) { + sb.append("\n\t\t").append(e.toString()).append(",\n"); + } + sb.append("\t\t]\n"); + } + sb.append("}\n"); + LOG.debug(sb.toString()); + } + } + + /** + * Generate combination of devices and its cost. + * costTable + * */ + private void buildCostTable( + Map, Integer>>> cTable, + Set ltfDevices) { + Device[] deviceList = new Device[ltfDevices.size()]; + ltfDevices.toArray(deviceList); + generateAllDeviceCombination(cTable, deviceList, deviceList.length); + } + + /** + * For every possible combination of i elements. + * We generate a map whose key is the combination, value is cost. + */ + private void generateAllDeviceCombination( + Map, Integer>>> cTable, + Device[] allDevices, int n) { + // allocated devices count range from 1 to n-1 + for (int i = 2; i < n; i++) { + Map, Integer> combinationToCost = + new HashMap<>(); + buildCombination(combinationToCost, allDevices, n, i); + // sort the map entry by cost ascending order + List, Integer>> listSortedByCost = + new LinkedList<>(combinationToCost.entrySet()); + Collections.sort(listSortedByCost, + (o1, o2) -> (o1.getValue()).compareTo(o2.getValue())); + cTable.put(i, listSortedByCost); + } + } + + private void buildCombination(Map, Integer> combinationToCost, + Device[] allDevices, int n, int r) { + // A temporary list to store all combination one by one + Device[] subDeviceList = new Device[r]; + combinationRecursive(combinationToCost, allDevices, subDeviceList, + 0, n - 1, 0, r); + } + + /** + * Populate combination to cost map recursively. + * + * @param cTc combinationToCost map. + * The key is device set, the value is cost + * @param allDevices all devices used to assign value to subDevicelist + * @param subDeviceList store a subset of devices temporary + * @param start start index in the allDevices + * @param end last index in the allDevices + * @param index dynamic index in subDeviceList need to be assigned + * @param r the length of the subDeviceList + */ + void combinationRecursive(Map, Integer> cTc, + Device[] allDevices, Device[] subDeviceList, + int start, int end, int index, int r) { + // sub device list's length is ready to compute the cost + if (index == r) { + Set oneSet = new TreeSet<>(Arrays.asList(subDeviceList)); + int cost = computeCostOfDevices(subDeviceList); + cTc.put(oneSet, cost); + return; + } + for (int i = start; i <= end; i++) { + subDeviceList[index] = allDevices[i]; + combinationRecursive(cTc, allDevices, subDeviceList, + i + 1, end, index + 1, r); + } + } + + /** + * The cost function used to calculate costs of a sub set of devices. + * It calculate link weight of each pair in non-duplicated combination of + * devices. + */ + @VisibleForTesting + public int computeCostOfDevices(Device[] devices) { + int cost = 0; + String gpuIndex0; + String gpuIndex1; + for (int i = 0; i < devices.length; i++) { + gpuIndex0 = String.valueOf(devices[i].getMinorNumber()); + for (int j = i + 1; j < devices.length; j++) { + gpuIndex1 = String.valueOf(devices[j].getMinorNumber()); + cost += this.devicePairToWeight.get(gpuIndex0 + "-" + gpuIndex1); + } + } + return cost; + } + + /** + * Topology Aware schedule algorithm. + * It doesn't consider CPU affinity or NUMA or bus bandwidths. + * It support two plicy: "spread" and "pack" which can be set by container's + * environment variable. Use pack by default which means prefer the faster + * GPU-GPU. "Spread" means prefer the faster CPU-GPU. + * It can potentially be extend to take GPU attribute like GPU chip memory + * into consideration. + * */ + @VisibleForTesting + public void topologyAwareSchedule(Set allocation, int count, + Map envs, + Set availableDevices, + Map, Integer>>> cTable) { + int num = 0; + String policy = envs.get(TOPOLOGY_POLICY_ENV_KEY); + if (policy == null) { + policy = TOPOLOGY_POLICY_PACK; + } + + /** + * Get combinations from costTable given the count of device want to + * allocate. + * */ + if (cTable == null) { + LOG.error("No cost table initialized!"); + return; + } + List, Integer>> combinationsToCost = + cTable.get(count); + Iterator, Integer>> iterator = + combinationsToCost.iterator(); + // the container needs spread policy + if (policy.equalsIgnoreCase(TOPOLOGY_POLICY_SPREAD)) { + // loop from high cost to low cost + iterator = ((LinkedList) combinationsToCost).descendingIterator(); + } + while (iterator.hasNext()) { + Map.Entry, Integer> element = iterator.next(); + if (availableDevices.containsAll(element.getKey())) { + allocation.addAll(element.getKey()); + LOG.info("Topology scheduler allocated: " + allocation); + return; + } + } + LOG.error("Unknown error happened in topology scheduler"); + } + + @VisibleForTesting + public void basicSchedule(Set allocation, int count, + Set availableDevices) { + // Basic scheduling + // allocate all available + if (count == availableDevices.size()) { + allocation.addAll(availableDevices); + return; + } + int number = 0; + for (Device d : availableDevices) { + allocation.add(d); + number++; + if (number == count) { + break; + } + } + } + + /** + * A typical sample topo output: + * GPU0 GPU1 GPU2 GPU3 CPU Affinity + * GPU0 X PHB SOC SOC 0-31 + * GPU1 PHB X SOC SOC 0-31 + * GPU2 SOC SOC X PHB 0-31 + * GPU3 SOC SOC PHB X 0-31 + * + * + * Legend: + * + * X = Self + * SOC = Connection traversing PCIe as well as the SMP link between + * CPU sockets(e.g. QPI) + * PHB = Connection traversing PCIe as well as a PCIe Host Bridge + * (typically the CPU) + * PXB = Connection traversing multiple PCIe switches + * (without traversing the PCIe Host Bridge) + * PIX = Connection traversing a single PCIe switch + * NV# = Connection traversing a bonded set of # NVLinks」 + * */ + public void parseTopo(String topo, + Map deviceLinkToWeight) { + String[] lines = topo.split("\n"); + int rowMinor; + int colMinor; + String legend; + String tempType; + for (String oneLine : lines) { + oneLine = oneLine.trim(); + if (oneLine.isEmpty()) { + continue; + } + // To the end. No more metrics info + if (oneLine.startsWith("Legend")) { + break; + } + // Skip header + if (oneLine.contains("Affinity")) { + continue; + } + String[] tokens = oneLine.split(("\\s+")); + String name = tokens[0]; + rowMinor = Integer.parseInt(name.substring(name.lastIndexOf("U") + 1)); + for (int i = 1; i < tokens.length; i++) { + tempType = tokens[i]; + colMinor = i - 1; + // self, skip + if (tempType.equals("X")) { + continue; + } + if (tempType.equals("SOC") || tempType.equals("SYS")) { + populateGraphEdgeWeight(DeviceLinkType.P2PLinkCrossCPUSocket, + rowMinor, colMinor, deviceLinkToWeight); + continue; + } + if (tempType.equals("PHB") || tempType.equals("NODE")) { + populateGraphEdgeWeight(DeviceLinkType.P2PLinkSameCPUSocket, + rowMinor, colMinor, deviceLinkToWeight); + continue; + } + if (tempType.equals("PXB")) { + populateGraphEdgeWeight(DeviceLinkType.P2PLinkMultiSwitch, + rowMinor, colMinor, deviceLinkToWeight); + continue; + } + if (tempType.equals("PIX")) { + populateGraphEdgeWeight(DeviceLinkType.P2PLinkSingleSwitch, + rowMinor, colMinor, deviceLinkToWeight); + continue; + } + if (tempType.equals("NV1")) { + populateGraphEdgeWeight(DeviceLinkType.P2PLinkNVLink1, + rowMinor, colMinor, deviceLinkToWeight); + continue; + } + if (tempType.equals("NV2")) { + populateGraphEdgeWeight(DeviceLinkType.P2PLinkNVLink2, + rowMinor, colMinor, deviceLinkToWeight); + continue; + } + if (tempType.equals("NV3")) { + populateGraphEdgeWeight(DeviceLinkType.P2PLinkNVLink3, + rowMinor, colMinor, deviceLinkToWeight); + continue; + } + if (tempType.equals("NV4")) { + populateGraphEdgeWeight(DeviceLinkType.P2PLinkNVLink4, + rowMinor, colMinor, deviceLinkToWeight); + continue; + } + if (tempType.equals("NV5")) { + populateGraphEdgeWeight(DeviceLinkType.P2PLinkNVLink5, + rowMinor, colMinor, deviceLinkToWeight); + continue; + } + if (tempType.equals("NV6")) { + populateGraphEdgeWeight(DeviceLinkType.P2PLinkNVLink6, + rowMinor, colMinor, deviceLinkToWeight); + continue; + } + if (tempType.equals("NV7")) { + populateGraphEdgeWeight(DeviceLinkType.P2PLinkNVLink7, + rowMinor, colMinor, deviceLinkToWeight); + continue; + } + if (tempType.equals("NV8")) { + populateGraphEdgeWeight(DeviceLinkType.P2PLinkNVLink8, + rowMinor, colMinor, deviceLinkToWeight); + continue; + } + if (tempType.equals("NV9")) { + populateGraphEdgeWeight(DeviceLinkType.P2PLinkNVLink9, + rowMinor, colMinor, deviceLinkToWeight); + continue; + } + } // end one line handling + } + } + + private void populateGraphEdgeWeight( + DeviceLinkType linkType, + int leftVertex, + int rightVertex, + Map deviceLinkToWeight) { + deviceLinkToWeight.put(leftVertex + "-" + rightVertex, + linkType.getWeight()); + } + + /** + * Different type of link. + * The weight of each link is a relative value. + * The higher weight, the higher cost between the GPUs + * */ + public enum DeviceLinkType { + /** + * For Nvdia GPU NVLink. + * */ + P2PLinkNVLink9(10), + P2PLinkNVLink8(20), + P2PLinkNVLink7(30), + P2PLinkNVLink6(40), + P2PLinkNVLink5(50), + P2PLinkNVLink4(60), + P2PLinkNVLink3(70), + P2PLinkNVLink2(80), + P2PLinkNVLink1(90), + + /** + * Connected to same CPU (Same NUMA node). + * */ + P2PLinkSameCPUSocket(200), + + /** + * Cross CPU through socket-level link (e.g. QPI). + * Usually cross NUMA node + * */ + P2PLinkCrossCPUSocket(300), + + /** + * Just need to traverse one PCIe switch to talk. + * */ + P2PLinkSingleSwitch(600), + + /** + * Need to traverse multiple PCIe switch to talk. + * */ + P2PLinkMultiSwitch(1200); + + // A higher link level means slower communication. + private int weight; + + public int getWeight() { + return weight; + } + + DeviceLinkType(int w) { + this.weight = w; + } + } + /** * A shell wrapper class easy for test. * */ @@ -189,6 +656,13 @@ public String getMajorMinorInfo(String devName) throws IOException { return shexec.getOutput(); } + // Get the topology metrics info from nvdia-smi + public String getTopologyInfo() throws IOException { + return Shell.execCommand(environment, + new String[]{pathOfGpuBinary, "topo", + "-m"}, MAX_EXEC_TIMEOUT_MS); + } + public void searchBinary() throws Exception { if (pathOfGpuBinary != null) { LOG.info("Skip searching, the nvidia gpu binary is already set: " @@ -228,8 +702,8 @@ public void searchBinary() throws Exception { } @VisibleForTesting - public void setPathOfGpuBinary(String pathOfGpuBinary) { - this.pathOfGpuBinary = pathOfGpuBinary; + public void setPathOfGpuBinary(String pOfGpuBinary) { + this.pathOfGpuBinary = pOfGpuBinary; } @VisibleForTesting @@ -237,4 +711,20 @@ public void setShellExecutor( NvidiaCommandExecutor shellExecutor) { this.shellExecutor = shellExecutor; } + + @VisibleForTesting + public boolean isTopoInitialized() { + return topoInitialized; + } + + @VisibleForTesting + public Map, Integer>>> getCostTable() { + return costTable; + } + + @VisibleForTesting + public Map getDevicePairToWeight() { + return devicePairToWeight; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DeviceMappingManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DeviceMappingManager.java index 9fcbf93cd3..b620620dc9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DeviceMappingManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/DeviceMappingManager.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.deviceframework; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import org.apache.commons.logging.Log; @@ -192,7 +193,7 @@ private synchronized DeviceAllocation internalAssignDevices( DevicePluginScheduler dps = devicePluginSchedulers.get(resourceName); // Prefer DevicePluginScheduler logic pickAndDoSchedule(allowedDevices, usedDevices, assignedDevices, - containerId, requestedDeviceCount, resourceName, dps); + container, requestedDeviceCount, resourceName, dps); // Record in state store if we allocated anything if (!assignedDevices.isEmpty()) { @@ -310,9 +311,11 @@ private long getReleasingDevices(String resourceName) { * */ private void pickAndDoSchedule(Set allowed, Map used, Set assigned, - ContainerId containerId, int count, String resourceName, - DevicePluginScheduler dps) throws ResourceHandlerException { - + Container c, int count, String resourceName, + DevicePluginScheduler dps) + throws ResourceHandlerException { + ContainerId containerId = c.getContainerId(); + Map env = c.getLaunchContext().getEnvironment(); if (null == dps) { if (LOG.isDebugEnabled()) { LOG.debug("Customized device plugin scheduler is preferred " @@ -331,7 +334,8 @@ private void pickAndDoSchedule(Set allowed, // Pass in unmodifiable set Set dpsAllocated = dps.allocateDevices( Sets.difference(allowed, used.keySet()), - count); + count, + ImmutableMap.copyOf(env)); if (dpsAllocated.size() != count) { throw new ResourceHandlerException(dps.getClass() + " should allocate " + count diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nvidia/TestNvidiaGPUPluginForRuntimeV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nvidia/TestNvidiaGPUPluginForRuntimeV2.java new file mode 100644 index 0000000000..595ee08a5e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nvidia/TestNvidiaGPUPluginForRuntimeV2.java @@ -0,0 +1,848 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.com.nvidia; + +import org.apache.commons.lang3.ArrayUtils; +import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.Device; +import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DeviceRuntimeSpec; +import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.YarnRuntimeType; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anySet; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Test case for NvidiaGPUPluginForRuntimeV2 device plugin. + * */ +public class TestNvidiaGPUPluginForRuntimeV2 { + + private static final Logger LOG = + LoggerFactory.getLogger(TestNvidiaGPUPluginForRuntimeV2.class); + + @Test + public void testGetNvidiaDevices() throws Exception { + NvidiaGPUPluginForRuntimeV2.NvidiaCommandExecutor mockShell = + mock(NvidiaGPUPluginForRuntimeV2.NvidiaCommandExecutor.class); + String deviceInfoShellOutput = + "0, 00000000:04:00.0\n" + + "1, 00000000:82:00.0"; + String majorMinorNumber0 = "c3:0"; + String majorMinorNumber1 = "c3:1"; + when(mockShell.getDeviceInfo()).thenReturn(deviceInfoShellOutput); + when(mockShell.getMajorMinorInfo("nvidia0")) + .thenReturn(majorMinorNumber0); + when(mockShell.getMajorMinorInfo("nvidia1")) + .thenReturn(majorMinorNumber1); + NvidiaGPUPluginForRuntimeV2 plugin = new NvidiaGPUPluginForRuntimeV2(); + plugin.setShellExecutor(mockShell); + plugin.setPathOfGpuBinary("/fake/nvidia-smi"); + + Set expectedDevices = new TreeSet<>(); + expectedDevices.add(Device.Builder.newInstance() + .setId(0).setHealthy(true) + .setBusID("00000000:04:00.0") + .setDevPath("/dev/nvidia0") + .setMajorNumber(195) + .setMinorNumber(0).build()); + expectedDevices.add(Device.Builder.newInstance() + .setId(1).setHealthy(true) + .setBusID("00000000:82:00.0") + .setDevPath("/dev/nvidia1") + .setMajorNumber(195) + .setMinorNumber(1).build()); + Set devices = plugin.getDevices(); + Assert.assertEquals(expectedDevices, devices); + } + + @Test + public void testOnDeviceAllocated() throws Exception { + NvidiaGPUPluginForRuntimeV2 plugin = new NvidiaGPUPluginForRuntimeV2(); + Set allocatedDevices = new TreeSet<>(); + + DeviceRuntimeSpec spec = plugin.onDevicesAllocated(allocatedDevices, + YarnRuntimeType.RUNTIME_DEFAULT); + Assert.assertNull(spec); + + // allocate one device + allocatedDevices.add(Device.Builder.newInstance() + .setId(0).setHealthy(true) + .setBusID("00000000:04:00.0") + .setDevPath("/dev/nvidia0") + .setMajorNumber(195) + .setMinorNumber(0).build()); + spec = plugin.onDevicesAllocated(allocatedDevices, + YarnRuntimeType.RUNTIME_DOCKER); + Assert.assertEquals("nvidia", spec.getContainerRuntime()); + Assert.assertEquals("0", spec.getEnvs().get("NVIDIA_VISIBLE_DEVICES")); + + // two device allowed + allocatedDevices.add(Device.Builder.newInstance() + .setId(0).setHealthy(true) + .setBusID("00000000:82:00.0") + .setDevPath("/dev/nvidia1") + .setMajorNumber(195) + .setMinorNumber(1).build()); + spec = plugin.onDevicesAllocated(allocatedDevices, + YarnRuntimeType.RUNTIME_DOCKER); + Assert.assertEquals("nvidia", spec.getContainerRuntime()); + Assert.assertEquals("0,1", spec.getEnvs().get("NVIDIA_VISIBLE_DEVICES")); + } + + private NvidiaGPUPluginForRuntimeV2 mockEightGPUPlugin() throws IOException { + String topoInfo = + "\tGPU0\tGPU1\tGPU2\tGPU3\tGPU4\tGPU5\tGPU6\tGPU7\tCPU Affinity\n" + + "GPU0\t X \tNV1\tNV1\tNV2\tNV2\tPHB\tPHB\tPHB\t0-63\n" + + "GPU1\tNV1\t X \tNV2\tNV1\tPHB\tNV2\tPHB\tPHB\t0-63\n" + + "GPU2\tNV1\tNV2\t X \tNV2\tPHB\tPHB\tNV1\tPHB\t0-63\n" + + "GPU3\tNV2\tNV1\tNV2\t X \tPHB\tPHB\tPHB\tNV1\t0-63\n" + + "GPU4\tNV2\tPHB\tPHB\tPHB\t X \tNV1\tNV1\tNV2\t0-63\n" + + "GPU5\tPHB\tNV2\tPHB\tPHB\tNV1\t X \tNV2\tNV1\t0-63\n" + + "GPU6\tPHB\tPHB\tNV1\tPHB\tNV1\tNV2\t X \tNV2\t0-63\n" + + "GPU7\tPHB\tPHB\tPHB\tNV1\tNV2\tNV1\tNV2\t X \t0-63\n" + + "\n" + + "Legend:\n" + + "\n" + + " X = Self\n" + + " SYS = Connection traversing PCIe as well as the SMP interconnect" + + " between NUMA nodes (e.g., QPI/UPI)\n" + + " NODE = Connection traversing PCIe as well as the interconnect" + + " between PCIe Host Bridges within a NUMA node\n" + + " PHB = Connection traversing PCIe as well as a PCIe Host Bridge" + + " (typically the CPU)\n" + + " PXB = Connection traversing multiple PCIe switches" + + " (without traversing the PCIe Host Bridge)\n" + + " PIX = Connection traversing a single PCIe switch\n" + + " NV# = Connection traversing a bonded set of # NVLinks\n"; + + String deviceInfoShellOutput = "0, 00000000:04:00.0\n" + + "1, 00000000:82:00.0\n" + + "2, 00000000:83:00.0\n" + + "3, 00000000:84:00.0\n" + + "4, 00000000:85:00.0\n" + + "5, 00000000:86:00.0\n" + + "6, 00000000:87:00.0\n" + + "7, 00000000:88:00.0"; + String majorMinorNumber0 = "c3:0"; + String majorMinorNumber1 = "c3:1"; + String majorMinorNumber2 = "c3:2"; + String majorMinorNumber3 = "c3:3"; + String majorMinorNumber4 = "c3:4"; + String majorMinorNumber5 = "c3:5"; + String majorMinorNumber6 = "c3:6"; + String majorMinorNumber7 = "c3:7"; + NvidiaGPUPluginForRuntimeV2.NvidiaCommandExecutor mockShell = + mock(NvidiaGPUPluginForRuntimeV2.NvidiaCommandExecutor.class); + when(mockShell.getDeviceInfo()).thenReturn(deviceInfoShellOutput); + when(mockShell.getMajorMinorInfo("nvidia0")) + .thenReturn(majorMinorNumber0); + when(mockShell.getMajorMinorInfo("nvidia1")) + .thenReturn(majorMinorNumber1); + when(mockShell.getMajorMinorInfo("nvidia2")) + .thenReturn(majorMinorNumber2); + when(mockShell.getMajorMinorInfo("nvidia3")) + .thenReturn(majorMinorNumber3); + when(mockShell.getMajorMinorInfo("nvidia4")) + .thenReturn(majorMinorNumber4); + when(mockShell.getMajorMinorInfo("nvidia5")) + .thenReturn(majorMinorNumber5); + when(mockShell.getMajorMinorInfo("nvidia6")) + .thenReturn(majorMinorNumber6); + when(mockShell.getMajorMinorInfo("nvidia7")) + .thenReturn(majorMinorNumber7); + when(mockShell.getTopologyInfo()).thenReturn(topoInfo); + when(mockShell.getDeviceInfo()).thenReturn(deviceInfoShellOutput); + + NvidiaGPUPluginForRuntimeV2 plugin = new NvidiaGPUPluginForRuntimeV2(); + plugin.setShellExecutor(mockShell); + plugin.setPathOfGpuBinary("/fake/nvidia-smi"); + return plugin; + } + + private NvidiaGPUPluginForRuntimeV2 mockFourGPUPlugin() throws IOException { + String topoInfo = "\tGPU0\tGPU1\tGPU2\tGPU3\tCPU Affinity\n" + + "GPU0\t X \tPHB\tSOC\tSOC\t0-31\n" + + "GPU1\tPHB\t X \tSOC\tSOC\t0-31\n" + + "GPU2\tSOC\tSOC\t X \tPHB\t0-31\n" + + "GPU3\tSOC\tSOC\tPHB\t X \t0-31\n" + + "\n" + + "\n" + + " Legend:\n" + + "\n" + + " X = Self\n" + + " SOC = Connection traversing PCIe as well as the SMP link between\n" + + " CPU sockets(e.g. QPI)\n" + + " PHB = Connection traversing PCIe as well as a PCIe Host Bridge\n" + + " (typically the CPU)\n" + + " PXB = Connection traversing multiple PCIe switches\n" + + " (without traversing the PCIe Host Bridge)\n" + + " PIX = Connection traversing a single PCIe switch\n" + + " NV# = Connection traversing a bonded set of # NVLinks"; + + String deviceInfoShellOutput = "0, 00000000:04:00.0\n" + + "1, 00000000:82:00.0\n" + + "2, 00000000:83:00.0\n" + + "3, 00000000:84:00.0"; + String majorMinorNumber0 = "c3:0"; + String majorMinorNumber1 = "c3:1"; + String majorMinorNumber2 = "c3:2"; + String majorMinorNumber3 = "c3:3"; + NvidiaGPUPluginForRuntimeV2.NvidiaCommandExecutor mockShell = + mock(NvidiaGPUPluginForRuntimeV2.NvidiaCommandExecutor.class); + when(mockShell.getDeviceInfo()).thenReturn(deviceInfoShellOutput); + when(mockShell.getMajorMinorInfo("nvidia0")) + .thenReturn(majorMinorNumber0); + when(mockShell.getMajorMinorInfo("nvidia1")) + .thenReturn(majorMinorNumber1); + when(mockShell.getMajorMinorInfo("nvidia2")) + .thenReturn(majorMinorNumber2); + when(mockShell.getMajorMinorInfo("nvidia3")) + .thenReturn(majorMinorNumber3); + when(mockShell.getTopologyInfo()).thenReturn(topoInfo); + when(mockShell.getDeviceInfo()).thenReturn(deviceInfoShellOutput); + + NvidiaGPUPluginForRuntimeV2 plugin = new NvidiaGPUPluginForRuntimeV2(); + plugin.setShellExecutor(mockShell); + plugin.setPathOfGpuBinary("/fake/nvidia-smi"); + return plugin; + } + + @Test + public void testTopologySchedulingWithPackPolicy() throws Exception { + NvidiaGPUPluginForRuntimeV2 plugin = mockFourGPUPlugin(); + NvidiaGPUPluginForRuntimeV2 spyPlugin = spy(plugin); + // cache the total devices + Set allDevices = spyPlugin.getDevices(); + // environment variable to use PACK policy + Map env = new HashMap<>(); + env.put(NvidiaGPUPluginForRuntimeV2.TOPOLOGY_POLICY_ENV_KEY, + NvidiaGPUPluginForRuntimeV2.TOPOLOGY_POLICY_PACK); + // Case 0. if available devices is less than 3, no topo scheduling needed + Set copyAvailableDevices = new TreeSet<>(allDevices); + Iterator iterator0 = copyAvailableDevices.iterator(); + iterator0.next(); + iterator0.remove(); + iterator0.next(); + iterator0.remove(); + // Case 0. allocate 1 device + reset(spyPlugin); + Set allocation = spyPlugin.allocateDevices(copyAvailableDevices, + 1, env); + Assert.assertEquals(allocation.size(), 1); + verify(spyPlugin).basicSchedule(anySet(), anyInt(), anySet()); + Assert.assertFalse(spyPlugin.isTopoInitialized()); + + // Case 1. allocate 1 device + reset(spyPlugin); + allocation = spyPlugin.allocateDevices(allDevices, 1, env); + // ensure no topology scheduling needed + Assert.assertEquals(allocation.size(), 1); + verify(spyPlugin).basicSchedule(anySet(), anyInt(), anySet()); + reset(spyPlugin); + // Case 2. allocate all available + allocation = spyPlugin.allocateDevices(allDevices, allDevices.size(), env); + Assert.assertEquals(allocation.size(), allDevices.size()); + verify(spyPlugin).basicSchedule(anySet(), anyInt(), anySet()); + // Case 3. allocate 2 devices + reset(spyPlugin); + int count = 2; + Map pairToWeight = spyPlugin.getDevicePairToWeight(); + allocation = spyPlugin.allocateDevices(allDevices, count, env); + Assert.assertEquals(allocation.size(), count); + // the costTable should be init and used topology scheduling + verify(spyPlugin).initCostTable(); + Assert.assertTrue(spyPlugin.isTopoInitialized()); + verify(spyPlugin).topologyAwareSchedule(anySet(), anyInt(), anyMap(), + anySet(), anyMap()); + Assert.assertEquals(allocation.size(), count); + Device[] allocatedDevices = + allocation.toArray(new Device[count]); + // Check weights + Assert.assertEquals(NvidiaGPUPluginForRuntimeV2.DeviceLinkType + .P2PLinkSameCPUSocket.getWeight(), + spyPlugin.computeCostOfDevices(allocatedDevices)); + // Case 4. allocate 3 devices + reset(spyPlugin); + count = 3; + allocation = spyPlugin.allocateDevices(allDevices, count, env); + Assert.assertEquals(allocation.size(), count); + // the costTable should be init and used topology scheduling + verify(spyPlugin, times(0)).initCostTable(); + Assert.assertTrue(spyPlugin.isTopoInitialized()); + verify(spyPlugin).topologyAwareSchedule(anySet(), anyInt(), anyMap(), + anySet(), anyMap()); + Assert.assertEquals(allocation.size(), count); + allocatedDevices = + allocation.toArray(new Device[count]); + // check weights + int expectedWeight = + NvidiaGPUPluginForRuntimeV2.DeviceLinkType + .P2PLinkSameCPUSocket.getWeight() + + 2 * NvidiaGPUPluginForRuntimeV2.DeviceLinkType + .P2PLinkCrossCPUSocket.getWeight(); + Assert.assertEquals(expectedWeight, + spyPlugin.computeCostOfDevices(allocatedDevices)); + // Case 5. allocate 2 GPUs from three available devices + reset(spyPlugin); + Iterator iterator = allDevices.iterator(); + iterator.next(); + // remove GPU0 + iterator.remove(); + count = 2; + allocation = spyPlugin.allocateDevices(allDevices, count, env); + Assert.assertEquals(allocation.size(), count); + // the costTable should be init and used topology scheduling + verify(spyPlugin, times(0)).initCostTable(); + Assert.assertTrue(spyPlugin.isTopoInitialized()); + verify(spyPlugin).topologyAwareSchedule(anySet(), anyInt(), anyMap(), + anySet(), anyMap()); + Assert.assertEquals(allocation.size(), count); + allocatedDevices = + allocation.toArray(new Device[count]); + // check weights + Assert.assertEquals(NvidiaGPUPluginForRuntimeV2.DeviceLinkType + .P2PLinkSameCPUSocket.getWeight(), + spyPlugin.computeCostOfDevices(allocatedDevices)); + // it should allocate GPU 2 and 3 + for (Device device : allocation) { + if (device.getMinorNumber() == 2) { + Assert.assertTrue(true); + } else if (device.getMinorNumber() == 3) { + Assert.assertTrue(true); + } else { + Assert.assertTrue("Should allocate GPU 2 and 3", false); + } + } + } + + @Test + public void testTopologySchedulingWithSpreadPolicy() throws Exception { + NvidiaGPUPluginForRuntimeV2 plugin = mockFourGPUPlugin(); + NvidiaGPUPluginForRuntimeV2 spyPlugin = spy(plugin); + // cache the total devices + Set allDevices = spyPlugin.getDevices(); + // environment variable to use PACK policy + Map env = new HashMap<>(); + env.put(NvidiaGPUPluginForRuntimeV2.TOPOLOGY_POLICY_ENV_KEY, + NvidiaGPUPluginForRuntimeV2.TOPOLOGY_POLICY_SPREAD); + // Case 1. allocate 1 device + Set allocation = spyPlugin.allocateDevices(allDevices, 1, env); + // ensure no topology scheduling needed + Assert.assertEquals(allocation.size(), 1); + verify(spyPlugin).basicSchedule(anySet(), anyInt(), anySet()); + reset(spyPlugin); + // Case 2. allocate all available + allocation = spyPlugin.allocateDevices(allDevices, allDevices.size(), env); + Assert.assertEquals(allocation.size(), allDevices.size()); + verify(spyPlugin).basicSchedule(anySet(), anyInt(), anySet()); + // Case 3. allocate 2 devices + reset(spyPlugin); + int count = 2; + Map pairToWeight = spyPlugin.getDevicePairToWeight(); + allocation = spyPlugin.allocateDevices(allDevices, count, env); + Assert.assertEquals(allocation.size(), count); + // the costTable should be init and used topology scheduling + verify(spyPlugin).initCostTable(); + Assert.assertTrue(spyPlugin.isTopoInitialized()); + verify(spyPlugin).topologyAwareSchedule(anySet(), anyInt(), anyMap(), + anySet(), anyMap()); + Assert.assertEquals(allocation.size(), count); + Device[] allocatedDevices = + allocation.toArray(new Device[count]); + // Check weights + Assert.assertEquals(NvidiaGPUPluginForRuntimeV2.DeviceLinkType + .P2PLinkCrossCPUSocket.getWeight(), + spyPlugin.computeCostOfDevices(allocatedDevices)); + // Case 4. allocate 3 devices + reset(spyPlugin); + count = 3; + allocation = spyPlugin.allocateDevices(allDevices, count, env); + Assert.assertEquals(allocation.size(), count); + // the costTable should be init and used topology scheduling + verify(spyPlugin, times(0)).initCostTable(); + Assert.assertTrue(spyPlugin.isTopoInitialized()); + verify(spyPlugin).topologyAwareSchedule(anySet(), anyInt(), anyMap(), + anySet(), anyMap()); + Assert.assertEquals(allocation.size(), count); + allocatedDevices = + allocation.toArray(new Device[count]); + // check weights + int expectedWeight = + NvidiaGPUPluginForRuntimeV2.DeviceLinkType + .P2PLinkSameCPUSocket.getWeight() + + 2 * NvidiaGPUPluginForRuntimeV2.DeviceLinkType + .P2PLinkCrossCPUSocket.getWeight(); + Assert.assertEquals(expectedWeight, + spyPlugin.computeCostOfDevices(allocatedDevices)); + // Case 5. allocate 2 GPUs from three available devices + reset(spyPlugin); + Iterator iterator = allDevices.iterator(); + iterator.next(); + // remove GPU0 + iterator.remove(); + count = 2; + allocation = spyPlugin.allocateDevices(allDevices, count, env); + Assert.assertEquals(allocation.size(), count); + // the costTable should be init and used topology scheduling + verify(spyPlugin, times(0)).initCostTable(); + Assert.assertTrue(spyPlugin.isTopoInitialized()); + verify(spyPlugin).topologyAwareSchedule(anySet(), anyInt(), anyMap(), + anySet(), anyMap()); + Assert.assertEquals(allocation.size(), count); + allocatedDevices = + allocation.toArray(new Device[count]); + // check weights + Assert.assertEquals(NvidiaGPUPluginForRuntimeV2.DeviceLinkType + .P2PLinkCrossCPUSocket.getWeight(), + spyPlugin.computeCostOfDevices(allocatedDevices)); + // it should allocate GPU 1 and 2 + for (Device device : allocation) { + if (device.getMinorNumber() == 0) { + Assert.assertTrue("Shouldn't allocate GPU 0", false); + } + } + } + + @Test + public void testCostTableWithNVlink() throws Exception { + NvidiaGPUPluginForRuntimeV2 plugin = mockEightGPUPlugin(); + NvidiaGPUPluginForRuntimeV2 spyPlugin = spy(plugin); + // verify the device pair to weight map + spyPlugin.initCostTable(); + Map devicePairToWeight = spyPlugin.getDevicePairToWeight(); + // 12 combinations when choose 2 GPUs from 8 respect the order. 8!/6! + Assert.assertEquals(56, devicePairToWeight.size()); + int sameCPUWeight = + NvidiaGPUPluginForRuntimeV2.DeviceLinkType + .P2PLinkSameCPUSocket.getWeight(); + int nv1Weight = + NvidiaGPUPluginForRuntimeV2.DeviceLinkType + .P2PLinkNVLink1.getWeight(); + int nv2Weight = + NvidiaGPUPluginForRuntimeV2.DeviceLinkType + .P2PLinkNVLink2.getWeight(); + + Assert.assertEquals(nv1Weight, (int)devicePairToWeight.get("0-1")); + Assert.assertEquals(nv1Weight, (int)devicePairToWeight.get("1-0")); + + Assert.assertEquals(nv2Weight, (int)devicePairToWeight.get("0-4")); + Assert.assertEquals(nv2Weight, (int)devicePairToWeight.get("4-0")); + + Assert.assertEquals(nv2Weight, (int)devicePairToWeight.get("0-3")); + Assert.assertEquals(nv2Weight, (int)devicePairToWeight.get("3-0")); + + Assert.assertEquals(sameCPUWeight, (int)devicePairToWeight.get("6-3")); + Assert.assertEquals(sameCPUWeight, (int)devicePairToWeight.get("3-6")); + + Assert.assertEquals(nv2Weight, (int)devicePairToWeight.get("6-7")); + Assert.assertEquals(nv2Weight, (int)devicePairToWeight.get("7-6")); + + Assert.assertEquals(nv1Weight, (int)devicePairToWeight.get("1-3")); + Assert.assertEquals(nv1Weight, (int)devicePairToWeight.get("3-1")); + + // verify cost Table + Map, Integer>>> costTable = + spyPlugin.getCostTable(); + Assert.assertNull(costTable.get(1)); + // C8:2 = 8!/2!/6! = 28 + Assert.assertEquals(28, costTable.get(2).size()); + // C8:4 = 8!/4!/4! = 70 + Assert.assertEquals(70, costTable.get(4).size()); + Assert.assertNull(costTable.get(8)); + + Set allDevices = spyPlugin.getDevices(); + Map env = new HashMap<>(); + env.put(NvidiaGPUPluginForRuntimeV2.TOPOLOGY_POLICY_ENV_KEY, + NvidiaGPUPluginForRuntimeV2.TOPOLOGY_POLICY_PACK); + spyPlugin.allocateDevices(allDevices, 3, env); + spyPlugin.allocateDevices(allDevices, 2, env); + } + + /** + * Test the key cost table used for topology scheduling. + * */ + @Test + public void testCostTable() throws IOException { + NvidiaGPUPluginForRuntimeV2 plugin = mockFourGPUPlugin(); + NvidiaGPUPluginForRuntimeV2 spyPlugin = spy(plugin); + // verify the device pair to weight map + spyPlugin.initCostTable(); + Map devicePairToWeight = spyPlugin.getDevicePairToWeight(); + // 12 combinations when choose 2 GPUs from 4 respect the order. 4!/2! + Assert.assertEquals(12, devicePairToWeight.size()); + int sameCPUWeight = + NvidiaGPUPluginForRuntimeV2.DeviceLinkType + .P2PLinkSameCPUSocket.getWeight(); + int crossCPUWeight = + NvidiaGPUPluginForRuntimeV2.DeviceLinkType + .P2PLinkCrossCPUSocket.getWeight(); + Assert.assertEquals(sameCPUWeight, (int)devicePairToWeight.get("0-1")); + Assert.assertEquals(sameCPUWeight, (int)devicePairToWeight.get("1-0")); + + Assert.assertEquals(crossCPUWeight, (int)devicePairToWeight.get("0-2")); + Assert.assertEquals(crossCPUWeight, (int)devicePairToWeight.get("2-0")); + + Assert.assertEquals(crossCPUWeight, (int)devicePairToWeight.get("0-3")); + Assert.assertEquals(crossCPUWeight, (int)devicePairToWeight.get("3-0")); + + Assert.assertEquals(crossCPUWeight, (int)devicePairToWeight.get("1-2")); + Assert.assertEquals(crossCPUWeight, (int)devicePairToWeight.get("2-1")); + + Assert.assertEquals(crossCPUWeight, (int)devicePairToWeight.get("1-3")); + Assert.assertEquals(crossCPUWeight, (int)devicePairToWeight.get("3-1")); + + Assert.assertEquals(sameCPUWeight, (int)devicePairToWeight.get("2-3")); + Assert.assertEquals(sameCPUWeight, (int)devicePairToWeight.get("3-2")); + + // verify cost Table + Map, Integer>>> costTable = + spyPlugin.getCostTable(); + Assert.assertNull(costTable.get(1)); + Assert.assertEquals(6, costTable.get(2).size()); + Assert.assertEquals(4, costTable.get(3).size()); + Assert.assertNull(costTable.get(4)); + } + /** + * Test GPU topology allocation. + * And analysis the GPU allocation's performance against the actual + * performance data using tensorflow benchmarks. + * https://github.com/tensorflow/benchmarks + * */ + @Test + public void testTopologySchedulingPerformanceWithPackPolicyWithNVLink() + throws Exception { + NvidiaGPUPluginForRuntimeV2 plugin = mockEightGPUPlugin(); + NvidiaGPUPluginForRuntimeV2 spyPlugin = spy(plugin); + Set allDevices = spyPlugin.getDevices(); + Map env = new HashMap<>(); + env.put(NvidiaGPUPluginForRuntimeV2.TOPOLOGY_POLICY_ENV_KEY, + NvidiaGPUPluginForRuntimeV2.TOPOLOGY_POLICY_PACK); + + /** + * Analyze performance against the real data. + * Get the topology scheduling algorithm's allocation's + * average performance boost against median imagePerSecond and minimum + * imagePerSecond in certain model and batch size combinations. + * And then calculate the average performance boost. + * The average performance boost against + * median value means topology scheduler's allocation can stably + * outperforms 50% of possible allocations. + * The average performance boost against min value means the average boost + * comparing to the worst allocations in various scenarios. Which is more + * beautiful number for public promotion. + * And also the analysis shows the best performance boost against median + * and min value. + * */ + ActualPerformanceReport report = new ActualPerformanceReport(); + report.readFromFile(); + ArrayList dataSet = + report.getDataSet(); + Assert.assertEquals(dataSet.size(), 2952); + String[] allModels = {"alexnet", "resnet50", "vgg16", "inception3"}; + int[] batchSizes = {32, 64, 128}; + int[] gpuCounts = {2, 3, 4, 5, 6, 7}; + float totalBoostAgainstMedian = 0; + int count = 0; + float maxBoostAgainstMedian = 0; + float totalBoostAgainstMin = 0; + float maxBoostAgainstMin = 0; + for (String model : allModels) { + float totalBoostAgainstMinCertainModel = 0; + float totalBoostAgainstMedianCertainModel = 0; + float maxBoostAgainstMinCertainModel = 0; + float maxBoostAgainstMedianCertainModel = 0; + int countOfEachModel = 0; + for (int bs : batchSizes) { + for (int gpuCount: gpuCounts) { + float bstAgainstMedian = calculatePerformanceBoostAgainstMedian( + report, model, bs, gpuCount, plugin, allDevices, env); + float bstAgainstMinimum = calculatePerformanceBoostAgainstMinimum( + report, model, bs, gpuCount, plugin, allDevices, env); + totalBoostAgainstMedian += bstAgainstMedian; + totalBoostAgainstMin += bstAgainstMinimum; + count++; + if (maxBoostAgainstMedian < bstAgainstMedian) { + maxBoostAgainstMedian = bstAgainstMedian; + } + if (maxBoostAgainstMin < bstAgainstMinimum) { + maxBoostAgainstMin = bstAgainstMinimum; + } + totalBoostAgainstMinCertainModel += bstAgainstMinimum; + totalBoostAgainstMedianCertainModel += bstAgainstMedian; + if (maxBoostAgainstMinCertainModel < bstAgainstMinimum) { + maxBoostAgainstMinCertainModel = bstAgainstMinimum; + } + if (maxBoostAgainstMedianCertainModel < bstAgainstMedian) { + maxBoostAgainstMedianCertainModel = bstAgainstMedian; + } + countOfEachModel++; + } + } + LOG.info("Model:{}, The best performance boost against median value is " + + "{}", model, maxBoostAgainstMedianCertainModel); + LOG.info("Model:{}, The aggregated average performance boost against " + + "median value is {}", + model, totalBoostAgainstMedianCertainModel/countOfEachModel); + LOG.info("Model:{}, The best performance boost against min value is {}", + model, maxBoostAgainstMinCertainModel); + LOG.info("Model:{}, The aggregated average performance boost against " + + "min value is {}", + model, totalBoostAgainstMinCertainModel/countOfEachModel); + } + LOG.info("For all, the best performance boost against median value is " + + maxBoostAgainstMedian); + LOG.info("For all, the aggregated average performance boost against median " + + "value is " + totalBoostAgainstMedian/count); + LOG.info("For all, the best performance boost against min value is " + + maxBoostAgainstMin); + LOG.info("For all, the aggregated average performance boost against min " + + "value is " + totalBoostAgainstMin/count); + } + + /** + * For gpuCount GPUs allocated by the topology algorithm, return + * its performance boost against the median value. + * + * */ + private float calculatePerformanceBoostAgainstMedian( + ActualPerformanceReport report, + String model, int bs, int gpuCount, + NvidiaGPUPluginForRuntimeV2 plugin, Set allDevice, + Map env) { + Set allocation = plugin.allocateDevices(allDevice, gpuCount, env); + String gpuAllocationString = convertAllocationToGpuString(allocation); + float[] metrics = report.getVariousImagePerSecond(model, bs, + gpuCount, gpuAllocationString); + return metrics[7]; + } + + /** + * For gpuCount GPUs allocated by the topology algorithm, return + * its performance boost against the minimum value. + * + * */ + private float calculatePerformanceBoostAgainstMinimum( + ActualPerformanceReport report, + String model, int bs, int gpuCount, + NvidiaGPUPluginForRuntimeV2 plugin, Set allDevice, + Map env) { + Set allocation = plugin.allocateDevices(allDevice, gpuCount, env); + String gpuAllocationString = convertAllocationToGpuString(allocation); + float[] metrics = report.getVariousImagePerSecond(model, bs, + gpuCount, gpuAllocationString); + return metrics[5]; + } + + private String convertAllocationToGpuString(Set allocation) { + StringBuilder sb = new StringBuilder(); + for (Device device : allocation) { + sb.append(device.getMinorNumber() + "_"); + } + return sb.toString().substring(0, sb.lastIndexOf("_")); + } + + /** + * Representation of the performance data report. + * */ + private class ActualPerformanceReport { + + private ArrayList dataSet = new ArrayList<>(); + + public ArrayList getDataSet() { + return dataSet; + } + + /** + * One line in the report. + * */ + private class DataRecord { + DataRecord(String model, int bs, String combination, float fps, + int count) { + this.batchSize = bs; + this.gpuCombination = combination; + this.gpuCount = count; + this.model = model; + this.imagePerSecond = fps; + } + + public String getModel() { + return model; + } + + public int getBatchSize() { + return batchSize; + } + + public String getGpuCombination() { + return gpuCombination; + } + + public float getImagePerSecond() { + return imagePerSecond; + } + + public int getGpuCount() { + return gpuCount; + } + + private String model; + private int batchSize; + private String gpuCombination; + private float imagePerSecond; + private int gpuCount; + } + + /** + * The file is a real performance report got from a 8 GPUs AWS instance. + * It contains every combination GPUs' training performance of Tensorflow + * benchmark. + * The columns are the model name, batch size, gpu ids and imagesPerSecond + * */ + public void readFromFile() { + String csvReportFilePath = getClass().getClassLoader() + .getResource("tensorflow-bench-result-for-GPU.csv").getFile(); + BufferedReader br = null; + String line = ""; + try { + br = new BufferedReader(new FileReader(csvReportFilePath)); + String model; + int batchSize; + String gpuCombination; + float imagePerSecond; + int gpuCount; + while ((line = br.readLine()) != null) { + // skip the licence content + if (line.startsWith("#")) { + continue; + } + String[] tokens = line.replaceAll("\"", "").split(","); + if (tokens.length != 4) { + LOG.error("unexpected performance data format!"); + break; + } + model = tokens[0]; + batchSize = Integer.parseInt(tokens[1].trim()); + gpuCombination = tokens[2]; + imagePerSecond = Float.parseFloat(tokens[3]); + gpuCount = getGpuCount(gpuCombination); + this.dataSet.add(new DataRecord(model, batchSize, gpuCombination, + imagePerSecond, gpuCount)); + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (br != null) { + try { + br.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } // end finally + } + + /** + * Return the maximum, minimum, mean and median performance for model & + * bs & gpuCount. And the imagePerSecond for model & bs & gpuCount & + * gpuCombinations. And imagePerSecond performance boost comparing to + * minimum, mean and media value. + * */ + private float[] getVariousImagePerSecond(String model, int bs, + int gpuCount, String gpuCombinations) { + float[] result = new float[8]; + float max = 0; + float min = Float.MAX_VALUE; + float sum = 0; + int count = 0; + float wantedImagePerSecond = 0; + float currentImagePerSecond; + ArrayList allFps = new ArrayList<>(); + for (DataRecord dr : getDataSet()) { + currentImagePerSecond = dr.getImagePerSecond(); + if (dr.batchSize == bs + && model.equals(dr.getModel()) + && gpuCount == dr.getGpuCount()) { + sum += currentImagePerSecond; + count++; + if (max < currentImagePerSecond) { + max = currentImagePerSecond; + } + if (min > currentImagePerSecond) { + min = currentImagePerSecond; + } + if (gpuCombinations.equals(dr.getGpuCombination())) { + wantedImagePerSecond = dr.getImagePerSecond(); + } + allFps.add(dr.getImagePerSecond()); + } + } + float median = getMedian(allFps); + float mean = sum/count; + result[0] = max; + result[1] = min; + result[2] = mean; + result[3] = median; + result[4] = wantedImagePerSecond; + result[5] = wantedImagePerSecond/min - 1; + result[6] = wantedImagePerSecond/mean - 1; + result[7] = wantedImagePerSecond/median - 1; + return result; + } + + private float getMedian(ArrayList allFps) { + float[] all = ArrayUtils.toPrimitive(allFps.toArray(new Float[0]), 0); + Arrays.sort(all); + float median; + int size = all.length; + if (allFps.size() % 2 == 0) { + median = (all[size/2] + all[size/2 - 1])/2; + } else { + median = all[size/2]; + } + return median; + } + + private int getGpuCount(String gpuCombination) { + String[] tokens = gpuCombination.split("_"); + return tokens.length; + } + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin1.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin1.java index 12f106411b..69736fd969 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin1.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/FakeTestDevicePlugin1.java @@ -20,6 +20,7 @@ import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.*; +import java.util.Map; import java.util.Set; import java.util.TreeSet; @@ -62,7 +63,7 @@ public void onDevicesReleased(Set allocatedDevices) { @Override public Set allocateDevices(Set availableDevices, - int count) { + int count, Map env) { Set allocated = new TreeSet(); int number = 0; for (Device d : availableDevices) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/TestDevicePluginAdapter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/TestDevicePluginAdapter.java index 75668f2a5d..2a64522a49 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/TestDevicePluginAdapter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/deviceframework/TestDevicePluginAdapter.java @@ -74,10 +74,11 @@ import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; - import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anySet; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.isA; @@ -571,7 +572,7 @@ public void testPreferPluginScheduler() throws IOException, YarnException { adapter.getDeviceResourceHandler().preStart(c1); // Use customized scheduler verify(spyPlugin, times(1)).allocateDevices( - any(Set.class), anyInt()); + anySet(), anyInt(), anyMap()); Assert.assertEquals(2, dmm.getAvailableDevices(resourceName)); Assert.assertEquals(1, @@ -995,7 +996,7 @@ public void onDevicesReleased(Set releasedDevices) { @Override public Set allocateDevices(Set availableDevices, - int count) { + int count, Map env) { Set allocated = new TreeSet<>(); int number = 0; for (Device d : availableDevices) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/nvidia/com/TestNvidiaGpuPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/nvidia/com/TestNvidiaGpuPlugin.java deleted file mode 100644 index 33154d84ea..0000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/nvidia/com/TestNvidiaGpuPlugin.java +++ /dev/null @@ -1,108 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.nvidia.com; - -import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.Device; -import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.DeviceRuntimeSpec; -import org.apache.hadoop.yarn.server.nodemanager.api.deviceplugin.YarnRuntimeType; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.com.nvidia.NvidiaGPUPluginForRuntimeV2; -import org.junit.Assert; -import org.junit.Test; - -import java.util.Set; -import java.util.TreeSet; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -/** - * Test case for Nvidia GPU device plugin. - * */ -public class TestNvidiaGpuPlugin { - - @Test - public void testGetNvidiaDevices() throws Exception { - NvidiaGPUPluginForRuntimeV2.NvidiaCommandExecutor mockShell = - mock(NvidiaGPUPluginForRuntimeV2.NvidiaCommandExecutor.class); - String deviceInfoShellOutput = - "0, 00000000:04:00.0\n" + - "1, 00000000:82:00.0"; - String majorMinorNumber0 = "c3:0"; - String majorMinorNumber1 = "c3:1"; - when(mockShell.getDeviceInfo()).thenReturn(deviceInfoShellOutput); - when(mockShell.getMajorMinorInfo("nvidia0")) - .thenReturn(majorMinorNumber0); - when(mockShell.getMajorMinorInfo("nvidia1")) - .thenReturn(majorMinorNumber1); - NvidiaGPUPluginForRuntimeV2 plugin = new NvidiaGPUPluginForRuntimeV2(); - plugin.setShellExecutor(mockShell); - plugin.setPathOfGpuBinary("/fake/nvidia-smi"); - - Set expectedDevices = new TreeSet<>(); - expectedDevices.add(Device.Builder.newInstance() - .setId(0).setHealthy(true) - .setBusID("00000000:04:00.0") - .setDevPath("/dev/nvidia0") - .setMajorNumber(195) - .setMinorNumber(0).build()); - expectedDevices.add(Device.Builder.newInstance() - .setId(1).setHealthy(true) - .setBusID("00000000:82:00.0") - .setDevPath("/dev/nvidia1") - .setMajorNumber(195) - .setMinorNumber(1).build()); - Set devices = plugin.getDevices(); - Assert.assertEquals(expectedDevices, devices); - } - - @Test - public void testOnDeviceAllocated() throws Exception { - NvidiaGPUPluginForRuntimeV2 plugin = new NvidiaGPUPluginForRuntimeV2(); - Set allocatedDevices = new TreeSet<>(); - - DeviceRuntimeSpec spec = plugin.onDevicesAllocated(allocatedDevices, - YarnRuntimeType.RUNTIME_DEFAULT); - Assert.assertNull(spec); - - // allocate one device - allocatedDevices.add(Device.Builder.newInstance() - .setId(0).setHealthy(true) - .setBusID("00000000:04:00.0") - .setDevPath("/dev/nvidia0") - .setMajorNumber(195) - .setMinorNumber(0).build()); - spec = plugin.onDevicesAllocated(allocatedDevices, - YarnRuntimeType.RUNTIME_DOCKER); - Assert.assertEquals("nvidia", spec.getContainerRuntime()); - Assert.assertEquals("0", spec.getEnvs().get("NVIDIA_VISIBLE_DEVICES")); - - // two device allowed - allocatedDevices.add(Device.Builder.newInstance() - .setId(0).setHealthy(true) - .setBusID("00000000:82:00.0") - .setDevPath("/dev/nvidia1") - .setMajorNumber(195) - .setMinorNumber(1).build()); - spec = plugin.onDevicesAllocated(allocatedDevices, - YarnRuntimeType.RUNTIME_DOCKER); - Assert.assertEquals("nvidia", spec.getContainerRuntime()); - Assert.assertEquals("0,1", spec.getEnvs().get("NVIDIA_VISIBLE_DEVICES")); - - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/tensorflow-bench-result-for-GPU.csv b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/tensorflow-bench-result-for-GPU.csv new file mode 100644 index 0000000000..0ff6f3d4ae --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/tensorflow-bench-result-for-GPU.csv @@ -0,0 +1,2963 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"alexnet","32","0_2","2910.87" +"resnet50","32","0_2","601.82" +"vgg16","32","0_2","382.83" +"inception3","32","0_2","392.53" +"alexnet","64","0_2","4539.27" +"resnet50","64","0_2","699.90" +"vgg16","64","0_2","428.95" +"inception3","64","0_2","435.53" +"alexnet","128","0_2","6319.08" +"resnet50","128","0_2","781.39" +"vgg16","128","0_2","454.85" +"inception3","128","0_2","456.94" +"alexnet","32","6_7","3922.30" +"resnet50","32","6_7","588.72" +"vgg16","32","6_7","406.52" +"inception3","32","6_7","380.24" +"alexnet","64","6_7","5693.85" +"resnet50","64","6_7","694.92" +"vgg16","64","6_7","437.44" +"inception3","64","6_7","429.96" +"alexnet","128","6_7","7226.03" +"resnet50","128","6_7","764.63" +"vgg16","128","6_7","451.29" +"inception3","128","6_7","454.66" +"alexnet","32","1_6","1552.58" +"resnet50","32","1_6","558.30" +"vgg16","32","1_6","315.01" +"inception3","32","1_6","375.01" +"alexnet","64","1_6","2829.13" +"resnet50","64","1_6","680.00" +"vgg16","64","1_6","382.46" +"inception3","64","1_6","427.11" +"alexnet","128","1_6","4448.61" +"resnet50","128","1_6","767.72" +"vgg16","128","1_6","425.83" +"inception3","128","1_6","453.58" +"alexnet","32","0_7","1559.17" +"resnet50","32","0_7","562.46" +"vgg16","32","0_7","313.64" +"inception3","32","0_7","379.64" +"alexnet","64","0_7","2861.61" +"resnet50","64","0_7","673.91" +"vgg16","64","0_7","381.18" +"inception3","64","0_7","424.98" +"alexnet","128","0_7","4453.04" +"resnet50","128","0_7","754.64" +"vgg16","128","0_7","419.01" +"inception3","128","0_7","447.85" +"alexnet","32","2_7","1565.90" +"resnet50","32","2_7","563.24" +"vgg16","32","2_7","310.85" +"inception3","32","2_7","377.70" +"alexnet","64","2_7","2837.95" +"resnet50","64","2_7","672.73" +"vgg16","64","2_7","377.73" +"inception3","64","2_7","428.27" +"alexnet","128","2_7","4462.88" +"resnet50","128","2_7","759.73" +"vgg16","128","2_7","418.70" +"inception3","128","2_7","453.87" +"alexnet","32","3_4","1564.28" +"resnet50","32","3_4","562.02" +"vgg16","32","3_4","314.93" +"inception3","32","3_4","379.44" +"alexnet","64","3_4","2852.29" +"resnet50","64","3_4","675.38" +"vgg16","64","3_4","382.05" +"inception3","64","3_4","429.81" +"alexnet","128","3_4","4496.05" +"resnet50","128","3_4","759.31" +"vgg16","128","3_4","423.16" +"inception3","128","3_4","455.16" +"alexnet","32","1_2","3907.10" +"resnet50","32","1_2","600.94" +"vgg16","32","1_2","409.16" +"inception3","32","1_2","390.87" +"alexnet","64","1_2","5738.23" +"resnet50","64","1_2","701.38" +"vgg16","64","1_2","446.87" +"inception3","64","1_2","437.65" +"alexnet","128","1_2","7260.26" +"resnet50","128","1_2","780.78" +"vgg16","128","1_2","466.06" +"inception3","128","1_2","464.19" +"alexnet","32","0_3","3896.77" +"resnet50","32","0_3","595.41" +"vgg16","32","0_3","409.77" +"inception3","32","0_3","390.42" +"alexnet","64","0_3","5754.24" +"resnet50","64","0_3","700.06" +"vgg16","64","0_3","443.89" +"inception3","64","0_3","435.08" +"alexnet","128","0_3","7257.56" +"resnet50","128","0_3","780.39" +"vgg16","128","0_3","458.60" +"inception3","128","0_3","457.85" +"alexnet","32","2_3","3900.87" +"resnet50","32","2_3","583.55" +"vgg16","32","2_3","406.56" +"inception3","32","2_3","380.68" +"alexnet","64","2_3","5718.20" +"resnet50","64","2_3","695.97" +"vgg16","64","2_3","442.44" +"inception3","64","2_3","431.38" +"alexnet","128","2_3","7240.75" +"resnet50","128","2_3","775.64" +"vgg16","128","2_3","464.23" +"inception3","128","2_3","457.18" +"alexnet","32","0_4","3840.64" +"resnet50","32","0_4","600.71" +"vgg16","32","0_4","410.09" +"inception3","32","0_4","390.52" +"alexnet","64","0_4","5718.76" +"resnet50","64","0_4","704.05" +"vgg16","64","0_4","443.72" +"inception3","64","0_4","435.45" +"alexnet","128","0_4","7237.24" +"resnet50","128","0_4","780.98" +"vgg16","128","0_4","459.58" +"inception3","128","0_4","456.95" +"alexnet","32","1_5","3883.02" +"resnet50","32","1_5","593.46" +"vgg16","32","1_5","410.63" +"inception3","32","1_5","388.79" +"alexnet","64","1_5","5715.07" +"resnet50","64","1_5","703.81" +"vgg16","64","1_5","445.57" +"inception3","64","1_5","436.14" +"alexnet","128","1_5","7264.90" +"resnet50","128","1_5","781.55" +"vgg16","128","1_5","463.32" +"inception3","128","1_5","462.65" +"alexnet","32","2_4","1559.84" +"resnet50","32","2_4","565.44" +"vgg16","32","2_4","317.92" +"inception3","32","2_4","379.84" +"alexnet","64","2_4","2857.86" +"resnet50","64","2_4","672.61" +"vgg16","64","2_4","382.30" +"inception3","64","2_4","429.39" +"alexnet","128","2_4","4479.28" +"resnet50","128","2_4","761.21" +"vgg16","128","2_4","425.00" +"inception3","128","2_4","455.17" +"alexnet","32","3_7","2912.16" +"resnet50","32","3_7","602.43" +"vgg16","32","3_7","379.59" +"inception3","32","3_7","391.27" +"alexnet","64","3_7","4548.95" +"resnet50","64","3_7","704.63" +"vgg16","64","3_7","421.16" +"inception3","64","3_7","437.74" +"alexnet","128","3_7","6292.67" +"resnet50","128","3_7","770.04" +"vgg16","128","3_7","443.30" +"inception3","128","3_7","459.13" +"alexnet","32","5_7","2903.06" +"resnet50","32","5_7","594.05" +"vgg16","32","5_7","378.23" +"inception3","32","5_7","388.99" +"alexnet","64","5_7","4528.86" +"resnet50","64","5_7","696.01" +"vgg16","64","5_7","422.22" +"inception3","64","5_7","432.35" +"alexnet","128","5_7","6298.93" +"resnet50","128","5_7","772.53" +"vgg16","128","5_7","443.27" +"inception3","128","5_7","460.91" +"alexnet","32","4_6","2909.87" +"resnet50","32","4_6","603.09" +"vgg16","32","4_6","382.92" +"inception3","32","4_6","391.43" +"alexnet","64","4_6","4529.73" +"resnet50","64","4_6","700.33" +"vgg16","64","4_6","427.43" +"inception3","64","4_6","431.65" +"alexnet","128","4_6","6313.76" +"resnet50","128","4_6","779.77" +"vgg16","128","4_6","451.37" +"inception3","128","4_6","455.94" +"alexnet","32","4_5","2909.74" +"resnet50","32","4_5","585.69" +"vgg16","32","4_5","382.49" +"inception3","32","4_5","379.25" +"alexnet","64","4_5","4539.81" +"resnet50","64","4_5","691.43" +"vgg16","64","4_5","427.21" +"inception3","64","4_5","428.58" +"alexnet","128","4_5","6287.64" +"resnet50","128","4_5","777.12" +"vgg16","128","4_5","452.45" +"inception3","128","4_5","449.28" +"alexnet","32","0_5","1556.36" +"resnet50","32","0_5","558.62" +"vgg16","32","0_5","313.98" +"inception3","32","0_5","380.35" +"alexnet","64","0_5","2848.95" +"resnet50","64","0_5","672.35" +"vgg16","64","0_5","382.56" +"inception3","64","0_5","426.09" +"alexnet","128","0_5","4453.50" +"resnet50","128","0_5","767.17" +"vgg16","128","0_5","426.69" +"inception3","128","0_5","446.46" +"alexnet","32","1_4","1551.61" +"resnet50","32","1_4","565.30" +"vgg16","32","1_4","317.38" +"inception3","32","1_4","380.51" +"alexnet","64","1_4","2837.42" +"resnet50","64","1_4","676.73" +"vgg16","64","1_4","381.76" +"inception3","64","1_4","433.22" +"alexnet","128","1_4","4458.53" +"resnet50","128","1_4","761.51" +"vgg16","128","1_4","422.85" +"inception3","128","1_4","459.32" +"alexnet","32","0_1","2918.37" +"resnet50","32","0_1","589.76" +"vgg16","32","0_1","381.40" +"inception3","32","0_1","386.02" +"alexnet","64","0_1","4552.76" +"resnet50","64","0_1","696.71" +"vgg16","64","0_1","425.30" +"inception3","64","0_1","426.34" +"alexnet","128","0_1","6295.42" +"resnet50","128","0_1","776.13" +"vgg16","128","0_1","450.08" +"inception3","128","0_1","453.51" +"alexnet","32","2_5","1558.34" +"resnet50","32","2_5","562.50" +"vgg16","32","2_5","314.97" +"inception3","32","2_5","377.27" +"alexnet","64","2_5","2849.74" +"resnet50","64","2_5","676.88" +"vgg16","64","2_5","382.05" +"inception3","64","2_5","426.25" +"alexnet","128","2_5","4453.20" +"resnet50","128","2_5","766.82" +"vgg16","128","2_5","429.61" +"inception3","128","2_5","452.35" +"alexnet","32","3_6","1563.11" +"resnet50","32","3_6","558.04" +"vgg16","32","3_6","314.65" +"inception3","32","3_6","377.83" +"alexnet","64","3_6","2859.37" +"resnet50","64","3_6","680.86" +"vgg16","64","3_6","384.07" +"inception3","64","3_6","426.29" +"alexnet","128","3_6","4454.93" +"resnet50","128","3_6","767.93" +"vgg16","128","3_6","426.04" +"inception3","128","3_6","456.97" +"alexnet","32","5_6","3921.50" +"resnet50","32","5_6","605.61" +"vgg16","32","5_6","409.56" +"inception3","32","5_6","390.59" +"alexnet","64","5_6","5666.52" +"resnet50","64","5_6","702.71" +"vgg16","64","5_6","442.91" +"inception3","64","5_6","434.45" +"alexnet","128","5_6","7265.78" +"resnet50","128","5_6","776.79" +"vgg16","128","5_6","462.54" +"inception3","128","5_6","463.67" +"alexnet","32","4_7","3827.15" +"resnet50","32","4_7","595.29" +"vgg16","32","4_7","407.49" +"inception3","32","4_7","392.00" +"alexnet","64","4_7","5736.73" +"resnet50","64","4_7","699.92" +"vgg16","64","4_7","437.92" +"inception3","64","4_7","433.93" +"alexnet","128","4_7","7241.76" +"resnet50","128","4_7","769.96" +"vgg16","128","4_7","451.85" +"inception3","128","4_7","461.71" +"alexnet","32","3_5","1545.52" +"resnet50","32","3_5","562.12" +"vgg16","32","3_5","314.16" +"inception3","32","3_5","377.18" +"alexnet","64","3_5","2821.83" +"resnet50","64","3_5","676.41" +"vgg16","64","3_5","381.80" +"inception3","64","3_5","427.90" +"alexnet","128","3_5","4442.79" +"resnet50","128","3_5","767.63" +"vgg16","128","3_5","423.39" +"inception3","128","3_5","454.34" +"alexnet","32","1_7","1553.76" +"resnet50","32","1_7","564.63" +"vgg16","32","1_7","316.17" +"inception3","32","1_7","377.83" +"alexnet","64","1_7","2835.24" +"resnet50","64","1_7","675.65" +"vgg16","64","1_7","377.14" +"inception3","64","1_7","430.26" +"alexnet","128","1_7","4447.93" +"resnet50","128","1_7","756.78" +"vgg16","128","1_7","418.48" +"inception3","128","1_7","451.21" +"alexnet","32","0_6","1562.12" +"resnet50","32","0_6","561.32" +"vgg16","32","0_6","317.29" +"inception3","32","0_6","379.52" +"alexnet","64","0_6","2846.76" +"resnet50","64","0_6","677.32" +"vgg16","64","0_6","383.31" +"inception3","64","0_6","425.92" +"alexnet","128","0_6","4468.00" +"resnet50","128","0_6","762.83" +"vgg16","128","0_6","427.33" +"inception3","128","0_6","447.95" +"alexnet","32","2_6","2906.72" +"resnet50","32","2_6","598.11" +"vgg16","32","2_6","380.97" +"inception3","32","2_6","389.59" +"alexnet","64","2_6","4551.68" +"resnet50","64","2_6","701.91" +"vgg16","64","2_6","427.35" +"inception3","64","2_6","434.31" +"alexnet","128","2_6","6309.10" +"resnet50","128","2_6","787.22" +"vgg16","128","2_6","454.29" +"inception3","128","2_6","466.53" +"alexnet","32","1_3","2900.56" +"resnet50","32","1_3","602.97" +"vgg16","32","1_3","382.61" +"inception3","32","1_3","388.50" +"alexnet","64","1_3","4508.73" +"resnet50","64","1_3","700.32" +"vgg16","64","1_3","427.10" +"inception3","64","1_3","438.16" +"alexnet","128","1_3","6313.84" +"resnet50","128","1_3","775.58" +"vgg16","128","1_3","449.99" +"inception3","128","1_3","462.13" +"alexnet","32","0_2_4","2079.28" +"resnet50","32","0_2_4","834.00" +"vgg16","32","0_2_4","491.30" +"inception3","32","0_2_4","588.88" +"alexnet","64","0_2_4","3952.09" +"resnet50","64","0_2_4","1001.06" +"vgg16","64","0_2_4","589.62" +"inception3","64","0_2_4","688.75" +"alexnet","128","0_2_4","6740.01" +"resnet50","128","0_2_4","1148.73" +"vgg16","128","0_2_4","649.41" +"inception3","128","0_2_4","737.81" +"alexnet","32","2_4_7","1749.71" +"resnet50","32","2_4_7","809.35" +"vgg16","32","2_4_7","487.14" +"inception3","32","2_4_7","580.21" +"alexnet","64","2_4_7","3290.68" +"resnet50","64","2_4_7","993.30" +"vgg16","64","2_4_7","583.16" +"inception3","64","2_4_7","683.73" +"alexnet","128","2_4_7","5716.17" +"resnet50","128","2_4_7","1123.65" +"vgg16","128","2_4_7","639.42" +"inception3","128","2_4_7","734.48" +"alexnet","32","1_5_6","3357.70" +"resnet50","32","1_5_6","837.21" +"vgg16","32","1_5_6","493.92" +"inception3","32","1_5_6","592.50" +"alexnet","64","1_5_6","5624.02" +"resnet50","64","1_5_6","998.43" +"vgg16","64","1_5_6","589.51" +"inception3","64","1_5_6","680.17" +"alexnet","128","1_5_6","8399.25" +"resnet50","128","1_5_6","1141.93" +"vgg16","128","1_5_6","648.14" +"inception3","128","1_5_6","749.69" +"alexnet","32","0_2_5","1798.27" +"resnet50","32","0_2_5","800.91" +"vgg16","32","0_2_5","325.78" +"inception3","32","0_2_5","570.46" +"alexnet","64","0_2_5","3366.59" +"resnet50","64","0_2_5","980.00" +"vgg16","64","0_2_5","380.18" +"inception3","64","0_2_5","672.99" +"alexnet","128","0_2_5","5723.50" +"resnet50","128","0_2_5","1131.75" +"vgg16","128","0_2_5","408.50" +"inception3","128","0_2_5","735.19" +"alexnet","32","1_2_3","4740.05" +"resnet50","32","1_2_3","856.45" +"vgg16","32","1_2_3","580.20" +"inception3","32","1_2_3","580.64" +"alexnet","64","1_2_3","7440.68" +"resnet50","64","1_2_3","1009.10" +"vgg16","64","1_2_3","648.38" +"inception3","64","1_2_3","675.79" +"alexnet","128","1_2_3","10143.99" +"resnet50","128","1_2_3","1139.79" +"vgg16","128","1_2_3","682.84" +"inception3","128","1_2_3","737.75" +"alexnet","32","1_5_7","3286.33" +"resnet50","32","1_5_7","837.23" +"vgg16","32","1_5_7","485.32" +"inception3","32","1_5_7","588.48" +"alexnet","64","1_5_7","5774.49" +"resnet50","64","1_5_7","1010.34" +"vgg16","64","1_5_7","585.79" +"inception3","64","1_5_7","678.16" +"alexnet","128","1_5_7","9127.83" +"resnet50","128","1_5_7","1123.67" +"vgg16","128","1_5_7","636.31" +"inception3","128","1_5_7","738.73" +"alexnet","32","1_2_6","3292.95" +"resnet50","32","1_2_6","827.72" +"vgg16","32","1_2_6","486.64" +"inception3","32","1_2_6","578.88" +"alexnet","64","1_2_6","5832.64" +"resnet50","64","1_2_6","1002.99" +"vgg16","64","1_2_6","588.80" +"inception3","64","1_2_6","681.65" +"alexnet","128","1_2_6","9206.24" +"resnet50","128","1_2_6","1146.75" +"vgg16","128","1_2_6","651.40" +"inception3","128","1_2_6","735.15" +"alexnet","32","4_5_6","3735.86" +"resnet50","32","4_5_6","860.70" +"vgg16","32","4_5_6","577.48" +"inception3","32","4_5_6","584.22" +"alexnet","64","4_5_6","6622.13" +"resnet50","64","4_5_6","1012.54" +"vgg16","64","4_5_6","642.65" +"inception3","64","4_5_6","671.19" +"alexnet","128","4_5_6","9360.12" +"resnet50","128","4_5_6","1152.27" +"vgg16","128","4_5_6","680.11" +"inception3","128","4_5_6","734.46" +"alexnet","32","2_4_5","1670.03" +"resnet50","32","2_4_5","798.93" +"vgg16","32","2_4_5","464.88" +"inception3","32","2_4_5","566.10" +"alexnet","64","2_4_5","3169.32" +"resnet50","64","2_4_5","969.10" +"vgg16","64","2_4_5","569.22" +"inception3","64","2_4_5","667.64" +"alexnet","128","2_4_5","5550.79" +"resnet50","128","2_4_5","1111.76" +"vgg16","128","2_4_5","634.23" +"inception3","128","2_4_5","737.61" +"alexnet","32","3_5_6","1743.34" +"resnet50","32","3_5_6","816.58" +"vgg16","32","3_5_6","490.76" +"inception3","32","3_5_6","576.92" +"alexnet","64","3_5_6","3291.97" +"resnet50","64","3_5_6","991.56" +"vgg16","64","3_5_6","588.85" +"inception3","64","3_5_6","682.14" +"alexnet","128","3_5_6","5785.63" +"resnet50","128","3_5_6","1133.08" +"vgg16","128","3_5_6","645.07" +"inception3","128","3_5_6","737.65" +"alexnet","32","0_2_3","3958.30" +"resnet50","32","0_2_3","849.42" +"vgg16","32","0_2_3","602.72" +"inception3","32","0_2_3","586.70" +"alexnet","64","0_2_3","6925.28" +"resnet50","64","0_2_3","1011.47" +"vgg16","64","0_2_3","660.90" +"inception3","64","0_2_3","683.35" +"alexnet","128","0_2_3","9573.54" +"resnet50","128","0_2_3","1150.96" +"vgg16","128","0_2_3","694.72" +"inception3","128","0_2_3","738.13" +"alexnet","32","1_4_5","2171.67" +"resnet50","32","1_4_5","836.67" +"vgg16","32","1_4_5","564.63" +"inception3","32","1_4_5","582.91" +"alexnet","64","1_4_5","4186.81" +"resnet50","64","1_4_5","1006.95" +"vgg16","64","1_4_5","638.46" +"inception3","64","1_4_5","685.53" +"alexnet","128","1_4_5","7010.65" +"resnet50","128","1_4_5","1129.47" +"vgg16","128","1_4_5","677.29" +"inception3","128","1_4_5","742.54" +"alexnet","32","3_4_6","1687.37" +"resnet50","32","3_4_6","811.04" +"vgg16","32","3_4_6","467.50" +"inception3","32","3_4_6","584.26" +"alexnet","64","3_4_6","3146.96" +"resnet50","64","3_4_6","1000.81" +"vgg16","64","3_4_6","576.32" +"inception3","64","3_4_6","680.45" +"alexnet","128","3_4_6","5577.87" +"resnet50","128","3_4_6","1128.46" +"vgg16","128","3_4_6","636.72" +"inception3","128","3_4_6","743.28" +"alexnet","32","2_3_7","3319.17" +"resnet50","32","2_3_7","833.32" +"vgg16","32","2_3_7","483.87" +"inception3","32","2_3_7","568.71" +"alexnet","64","2_3_7","5847.58" +"resnet50","64","2_3_7","1003.06" +"vgg16","64","2_3_7","583.99" +"inception3","64","2_3_7","666.59" +"alexnet","128","2_3_7","9140.01" +"resnet50","128","2_3_7","1130.20" +"vgg16","128","2_3_7","644.36" +"inception3","128","2_3_7","739.91" +"alexnet","32","1_6_7","1748.71" +"resnet50","32","1_6_7","802.21" +"vgg16","32","1_6_7","485.80" +"inception3","32","1_6_7","561.86" +"alexnet","64","1_6_7","3275.24" +"resnet50","64","1_6_7","971.70" +"vgg16","64","1_6_7","578.55" +"inception3","64","1_6_7","664.75" +"alexnet","128","1_6_7","5612.43" +"resnet50","128","1_6_7","1108.39" +"vgg16","128","1_6_7","628.97" +"inception3","128","1_6_7","725.26" +"alexnet","32","3_5_7","2133.23" +"resnet50","32","3_5_7","849.15" +"vgg16","32","3_5_7","553.56" +"inception3","32","3_5_7","600.06" +"alexnet","64","3_5_7","4050.74" +"resnet50","64","3_5_7","1016.88" +"vgg16","64","3_5_7","627.91" +"inception3","64","3_5_7","691.22" +"alexnet","128","3_5_7","7231.94" +"resnet50","128","3_5_7","1137.54" +"vgg16","128","3_5_7","664.70" +"inception3","128","3_5_7","740.57" +"alexnet","32","2_3_6","2234.63" +"resnet50","32","2_3_6","831.59" +"vgg16","32","2_3_6","480.03" +"inception3","32","2_3_6","572.81" +"alexnet","64","2_3_6","4169.79" +"resnet50","64","2_3_6","1007.82" +"vgg16","64","2_3_6","586.29" +"inception3","64","2_3_6","677.11" +"alexnet","128","2_3_6","6553.91" +"resnet50","128","2_3_6","1145.83" +"vgg16","128","2_3_6","652.69" +"inception3","128","2_3_6","732.84" +"alexnet","32","0_3_5","1937.14" +"resnet50","32","0_3_5","806.17" +"vgg16","32","0_3_5","328.01" +"inception3","32","0_3_5","570.30" +"alexnet","64","0_3_5","3578.00" +"resnet50","64","0_3_5","981.25" +"vgg16","64","0_3_5","379.53" +"inception3","64","0_3_5","666.86" +"alexnet","128","0_3_5","6052.57" +"resnet50","128","0_3_5","1118.65" +"vgg16","128","0_3_5","406.99" +"inception3","128","0_3_5","732.19" +"alexnet","32","2_3_5","1926.55" +"resnet50","32","2_3_5","805.04" +"vgg16","32","2_3_5","326.23" +"inception3","32","2_3_5","558.61" +"alexnet","64","2_3_5","3547.90" +"resnet50","64","2_3_5","977.43" +"vgg16","64","2_3_5","382.62" +"inception3","64","2_3_5","663.05" +"alexnet","128","2_3_5","5981.77" +"resnet50","128","2_3_5","1134.81" +"vgg16","128","2_3_5","409.66" +"inception3","128","2_3_5","741.46" +"alexnet","32","0_3_4","2226.94" +"resnet50","32","0_3_4","848.42" +"vgg16","32","0_3_4","493.18" +"inception3","32","0_3_4","587.78" +"alexnet","64","0_3_4","4219.32" +"resnet50","64","0_3_4","998.77" +"vgg16","64","0_3_4","592.35" +"inception3","64","0_3_4","681.22" +"alexnet","128","0_3_4","6774.41" +"resnet50","128","0_3_4","1135.61" +"vgg16","128","0_3_4","645.88" +"inception3","128","0_3_4","735.48" +"alexnet","32","2_3_4","1941.78" +"resnet50","32","2_3_4","804.32" +"vgg16","32","2_3_4","326.31" +"inception3","32","2_3_4","557.95" +"alexnet","64","2_3_4","3556.00" +"resnet50","64","2_3_4","990.46" +"vgg16","64","2_3_4","378.75" +"inception3","64","2_3_4","664.97" +"alexnet","128","2_3_4","6021.85" +"resnet50","128","2_3_4","1123.54" +"vgg16","128","2_3_4","405.19" +"inception3","128","2_3_4","739.83" +"alexnet","32","0_3_7","3294.85" +"resnet50","32","0_3_7","839.61" +"vgg16","32","0_3_7","486.59" +"inception3","32","0_3_7","584.84" +"alexnet","64","0_3_7","5667.63" +"resnet50","64","0_3_7","1000.57" +"vgg16","64","0_3_7","584.72" +"inception3","64","0_3_7","680.12" +"alexnet","128","0_3_7","9184.33" +"resnet50","128","0_3_7","1125.96" +"vgg16","128","0_3_7","633.98" +"inception3","128","0_3_7","735.43" +"alexnet","32","4_5_7","3427.28" +"resnet50","32","4_5_7","868.27" +"vgg16","32","4_5_7","567.99" +"inception3","32","4_5_7","578.78" +"alexnet","64","4_5_7","6198.05" +"resnet50","64","4_5_7","1002.96" +"vgg16","64","4_5_7","631.48" +"inception3","64","4_5_7","682.08" +"alexnet","128","4_5_7","8907.87" +"resnet50","128","4_5_7","1136.59" +"vgg16","128","4_5_7","666.25" +"inception3","128","4_5_7","732.44" +"alexnet","32","2_6_7","3219.58" +"resnet50","32","2_6_7","813.54" +"vgg16","32","2_6_7","481.11" +"inception3","32","2_6_7","567.79" +"alexnet","64","2_6_7","5938.84" +"resnet50","64","2_6_7","985.45" +"vgg16","64","2_6_7","580.06" +"inception3","64","2_6_7","666.98" +"alexnet","128","2_6_7","9450.72" +"resnet50","128","2_6_7","1107.06" +"vgg16","128","2_6_7","637.32" +"inception3","128","2_6_7","727.22" +"alexnet","32","1_2_5","2219.28" +"resnet50","32","1_2_5","833.15" +"vgg16","32","1_2_5","492.90" +"inception3","32","1_2_5","583.21" +"alexnet","64","1_2_5","4231.28" +"resnet50","64","1_2_5","1009.08" +"vgg16","64","1_2_5","591.90" +"inception3","64","1_2_5","685.51" +"alexnet","128","1_2_5","6729.82" +"resnet50","128","1_2_5","1139.33" +"vgg16","128","1_2_5","653.65" +"inception3","128","1_2_5","746.86" +"alexnet","32","0_3_6","1948.57" +"resnet50","32","0_3_6","806.41" +"vgg16","32","0_3_6","326.84" +"inception3","32","0_3_6","571.08" +"alexnet","64","0_3_6","3599.08" +"resnet50","64","0_3_6","992.91" +"vgg16","64","0_3_6","379.72" +"inception3","64","0_3_6","672.61" +"alexnet","128","0_3_6","6082.66" +"resnet50","128","0_3_6","1120.49" +"vgg16","128","0_3_6","407.15" +"inception3","128","0_3_6","723.71" +"alexnet","32","0_5_7","1677.06" +"resnet50","32","0_5_7","815.49" +"vgg16","32","0_5_7","465.59" +"inception3","32","0_5_7","577.20" +"alexnet","64","0_5_7","3110.85" +"resnet50","64","0_5_7","980.15" +"vgg16","64","0_5_7","570.96" +"inception3","64","0_5_7","673.43" +"alexnet","128","0_5_7","5506.22" +"resnet50","128","0_5_7","1124.58" +"vgg16","128","0_5_7","630.76" +"inception3","128","0_5_7","731.01" +"alexnet","32","4_6_7","3967.22" +"resnet50","32","4_6_7","848.63" +"vgg16","32","4_6_7","597.61" +"inception3","32","4_6_7","576.29" +"alexnet","64","4_6_7","6930.99" +"resnet50","64","4_6_7","1009.36" +"vgg16","64","4_6_7","649.87" +"inception3","64","4_6_7","677.69" +"alexnet","128","4_6_7","9519.65" +"resnet50","128","4_6_7","1131.50" +"vgg16","128","4_6_7","674.90" +"inception3","128","4_6_7","730.49" +"alexnet","32","1_3_4","1808.34" +"resnet50","32","1_3_4","809.24" +"vgg16","32","1_3_4","325.72" +"inception3","32","1_3_4","575.28" +"alexnet","64","1_3_4","3371.90" +"resnet50","64","1_3_4","983.28" +"vgg16","64","1_3_4","376.01" +"inception3","64","1_3_4","669.89" +"alexnet","128","1_3_4","5807.32" +"resnet50","128","1_3_4","1115.27" +"vgg16","128","1_3_4","404.52" +"inception3","128","1_3_4","738.10" +"alexnet","32","0_1_3","3429.59" +"resnet50","32","0_1_3","865.48" +"vgg16","32","0_1_3","575.10" +"inception3","32","0_1_3","587.80" +"alexnet","64","0_1_3","6184.53" +"resnet50","64","0_1_3","1014.01" +"vgg16","64","0_1_3","639.11" +"inception3","64","0_1_3","675.40" +"alexnet","128","0_1_3","8906.14" +"resnet50","128","0_1_3","1116.76" +"vgg16","128","0_1_3","675.15" +"inception3","128","0_1_3","730.13" +"alexnet","32","0_6_7","1760.17" +"resnet50","32","0_6_7","805.28" +"vgg16","32","0_6_7","486.54" +"inception3","32","0_6_7","565.58" +"alexnet","64","0_6_7","3305.71" +"resnet50","64","0_6_7","985.23" +"vgg16","64","0_6_7","584.18" +"inception3","64","0_6_7","664.87" +"alexnet","128","0_6_7","5716.68" +"resnet50","128","0_6_7","1120.53" +"vgg16","128","0_6_7","634.96" +"inception3","128","0_6_7","727.92" +"alexnet","32","1_3_5","2078.60" +"resnet50","32","1_3_5","830.27" +"vgg16","32","1_3_5","489.86" +"inception3","32","1_3_5","582.43" +"alexnet","64","1_3_5","3969.36" +"resnet50","64","1_3_5","1008.78" +"vgg16","64","1_3_5","589.81" +"inception3","64","1_3_5","679.00" +"alexnet","128","1_3_5","6736.64" +"resnet50","128","1_3_5","1148.53" +"vgg16","128","1_3_5","651.52" +"inception3","128","1_3_5","744.32" +"alexnet","32","2_5_7","1682.88" +"resnet50","32","2_5_7","815.30" +"vgg16","32","2_5_7","470.56" +"inception3","32","2_5_7","575.87" +"alexnet","64","2_5_7","3114.42" +"resnet50","64","2_5_7","983.11" +"vgg16","64","2_5_7","574.22" +"inception3","64","2_5_7","679.15" +"alexnet","128","2_5_7","5575.52" +"resnet50","128","2_5_7","1136.40" +"vgg16","128","2_5_7","629.84" +"inception3","128","2_5_7","735.68" +"alexnet","32","0_5_6","1750.56" +"resnet50","32","0_5_6","814.72" +"vgg16","32","0_5_6","488.15" +"inception3","32","0_5_6","573.77" +"alexnet","64","0_5_6","3302.69" +"resnet50","64","0_5_6","986.19" +"vgg16","64","0_5_6","587.78" +"inception3","64","0_5_6","672.10" +"alexnet","128","0_5_6","5770.32" +"resnet50","128","0_5_6","1117.08" +"vgg16","128","0_5_6","645.27" +"inception3","128","0_5_6","735.04" +"alexnet","32","3_4_5","1685.87" +"resnet50","32","3_4_5","793.21" +"vgg16","32","3_4_5","463.05" +"inception3","32","3_4_5","557.31" +"alexnet","64","3_4_5","3146.06" +"resnet50","64","3_4_5","972.87" +"vgg16","64","3_4_5","571.66" +"inception3","64","3_4_5","665.56" +"alexnet","128","3_4_5","5564.64" +"resnet50","128","3_4_5","1121.20" +"vgg16","128","3_4_5","635.35" +"inception3","128","3_4_5","735.31" +"alexnet","32","1_3_6","1819.65" +"resnet50","32","1_3_6","807.51" +"vgg16","32","1_3_6","326.14" +"inception3","32","1_3_6","575.25" +"alexnet","64","1_3_6","3359.29" +"resnet50","64","1_3_6","989.97" +"vgg16","64","1_3_6","380.75" +"inception3","64","1_3_6","671.00" +"alexnet","128","1_3_6","5788.55" +"resnet50","128","1_3_6","1133.81" +"vgg16","128","1_3_6","406.23" +"inception3","128","1_3_6","742.42" +"alexnet","32","2_5_6","2257.84" +"resnet50","32","2_5_6","831.32" +"vgg16","32","2_5_6","563.75" +"inception3","32","2_5_6","589.81" +"alexnet","64","2_5_6","4296.99" +"resnet50","64","2_5_6","1004.35" +"vgg16","64","2_5_6","638.20" +"inception3","64","2_5_6","695.85" +"alexnet","128","2_5_6","7224.86" +"resnet50","128","2_5_6","1141.31" +"vgg16","128","2_5_6","681.02" +"inception3","128","2_5_6","750.51" +"alexnet","32","1_4_7","1753.42" +"resnet50","32","1_4_7","819.05" +"vgg16","32","1_4_7","486.45" +"inception3","32","1_4_7","579.19" +"alexnet","64","1_4_7","3261.69" +"resnet50","64","1_4_7","987.71" +"vgg16","64","1_4_7","585.32" +"inception3","64","1_4_7","685.73" +"alexnet","128","1_4_7","5723.23" +"resnet50","128","1_4_7","1105.71" +"vgg16","128","1_4_7","635.61" +"inception3","128","1_4_7","732.11" +"alexnet","32","1_3_7","2922.14" +"resnet50","32","1_3_7","827.05" +"vgg16","32","1_3_7","484.53" +"inception3","32","1_3_7","583.22" +"alexnet","64","1_3_7","5438.01" +"resnet50","64","1_3_7","999.05" +"vgg16","64","1_3_7","578.10" +"inception3","64","1_3_7","678.90" +"alexnet","128","1_3_7","8173.49" +"resnet50","128","1_3_7","1117.18" +"vgg16","128","1_3_7","635.40" +"inception3","128","1_3_7","732.78" +"alexnet","32","5_6_7","4765.32" +"resnet50","32","5_6_7","848.15" +"vgg16","32","5_6_7","569.61" +"inception3","32","5_6_7","579.58" +"alexnet","64","5_6_7","7448.32" +"resnet50","64","5_6_7","1000.85" +"vgg16","64","5_6_7","633.16" +"inception3","64","5_6_7","674.75" +"alexnet","128","5_6_7","10125.70" +"resnet50","128","5_6_7","1124.29" +"vgg16","128","5_6_7","665.93" +"inception3","128","5_6_7","730.33" +"alexnet","32","1_4_6","1673.43" +"resnet50","32","1_4_6","821.14" +"vgg16","32","1_4_6","466.94" +"inception3","32","1_4_6","577.34" +"alexnet","64","1_4_6","3192.23" +"resnet50","64","1_4_6","989.40" +"vgg16","64","1_4_6","571.12" +"inception3","64","1_4_6","681.71" +"alexnet","128","1_4_6","5599.12" +"resnet50","128","1_4_6","1138.45" +"vgg16","128","1_4_6","636.74" +"inception3","128","1_4_6","736.75" +"alexnet","32","1_2_4","1943.31" +"resnet50","32","1_2_4","806.95" +"vgg16","32","1_2_4","327.60" +"inception3","32","1_2_4","572.14" +"alexnet","64","1_2_4","3572.71" +"resnet50","64","1_2_4","975.24" +"vgg16","64","1_2_4","380.38" +"inception3","64","1_2_4","673.11" +"alexnet","128","1_2_4","6094.53" +"resnet50","128","1_2_4","1116.32" +"vgg16","128","1_2_4","408.76" +"inception3","128","1_2_4","740.87" +"alexnet","32","1_2_7","1949.16" +"resnet50","32","1_2_7","809.69" +"vgg16","32","1_2_7","327.98" +"inception3","32","1_2_7","571.14" +"alexnet","64","1_2_7","3584.57" +"resnet50","64","1_2_7","981.31" +"vgg16","64","1_2_7","377.28" +"inception3","64","1_2_7","667.59" +"alexnet","128","1_2_7","6059.93" +"resnet50","128","1_2_7","1121.17" +"vgg16","128","1_2_7","403.07" +"inception3","128","1_2_7","728.24" +"alexnet","32","0_1_7","1783.54" +"resnet50","32","0_1_7","808.88" +"vgg16","32","0_1_7","323.93" +"inception3","32","0_1_7","560.39" +"alexnet","64","0_1_7","3321.22" +"resnet50","64","0_1_7","973.75" +"vgg16","64","0_1_7","375.68" +"inception3","64","0_1_7","659.72" +"alexnet","128","0_1_7","5771.63" +"resnet50","128","0_1_7","1106.24" +"vgg16","128","0_1_7","398.63" +"inception3","128","0_1_7","721.84" +"alexnet","32","0_1_2","3752.11" +"resnet50","32","0_1_2","863.84" +"vgg16","32","0_1_2","575.96" +"inception3","32","0_1_2","583.84" +"alexnet","64","0_1_2","6663.61" +"resnet50","64","0_1_2","1012.89" +"vgg16","64","0_1_2","646.64" +"inception3","64","0_1_2","667.07" +"alexnet","128","0_1_2","9387.99" +"resnet50","128","0_1_2","1145.90" +"vgg16","128","0_1_2","684.89" +"inception3","128","0_1_2","730.63" +"alexnet","32","0_1_6","1799.87" +"resnet50","32","0_1_6","803.27" +"vgg16","32","0_1_6","325.11" +"inception3","32","0_1_6","561.79" +"alexnet","64","0_1_6","3324.30" +"resnet50","64","0_1_6","990.06" +"vgg16","64","0_1_6","381.93" +"inception3","64","0_1_6","660.31" +"alexnet","128","0_1_6","5719.92" +"resnet50","128","0_1_6","1131.01" +"vgg16","128","0_1_6","407.74" +"inception3","128","0_1_6","732.38" +"alexnet","32","0_4_5","3271.89" +"resnet50","32","0_4_5","822.55" +"vgg16","32","0_4_5","478.86" +"inception3","32","0_4_5","565.98" +"alexnet","64","0_4_5","5772.92" +"resnet50","64","0_4_5","991.99" +"vgg16","64","0_4_5","586.74" +"inception3","64","0_4_5","672.47" +"alexnet","128","0_4_5","8929.89" +"resnet50","128","0_4_5","1139.48" +"vgg16","128","0_4_5","647.32" +"inception3","128","0_4_5","736.26" +"alexnet","32","0_2_6","2928.63" +"resnet50","32","0_2_6","830.14" +"vgg16","32","0_2_6","486.33" +"inception3","32","0_2_6","583.91" +"alexnet","64","0_2_6","5401.40" +"resnet50","64","0_2_6","1011.23" +"vgg16","64","0_2_6","585.51" +"inception3","64","0_2_6","675.72" +"alexnet","128","0_2_6","8149.77" +"resnet50","128","0_2_6","1116.18" +"vgg16","128","0_2_6","649.05" +"inception3","128","0_2_6","739.89" +"alexnet","32","3_4_7","2257.65" +"resnet50","32","3_4_7","847.49" +"vgg16","32","3_4_7","560.35" +"inception3","32","3_4_7","600.41" +"alexnet","64","3_4_7","4318.02" +"resnet50","64","3_4_7","1010.95" +"vgg16","64","3_4_7","626.70" +"inception3","64","3_4_7","689.80" +"alexnet","128","3_4_7","7249.43" +"resnet50","128","3_4_7","1132.47" +"vgg16","128","3_4_7","661.96" +"inception3","128","3_4_7","735.68" +"alexnet","32","0_1_5","3220.26" +"resnet50","32","0_1_5","835.88" +"vgg16","32","0_1_5","490.38" +"inception3","32","0_1_5","571.67" +"alexnet","64","0_1_5","5957.92" +"resnet50","64","0_1_5","1005.80" +"vgg16","64","0_1_5","591.77" +"inception3","64","0_1_5","664.13" +"alexnet","128","0_1_5","9536.47" +"resnet50","128","0_1_5","1129.10" +"vgg16","128","0_1_5","650.19" +"inception3","128","0_1_5","726.53" +"alexnet","32","3_6_7","2263.52" +"resnet50","32","3_6_7","828.83" +"vgg16","32","3_6_7","561.30" +"inception3","32","3_6_7","578.29" +"alexnet","64","3_6_7","4302.45" +"resnet50","64","3_6_7","989.18" +"vgg16","64","3_6_7","624.44" +"inception3","64","3_6_7","680.14" +"alexnet","128","3_6_7","7018.45" +"resnet50","128","3_6_7","1127.25" +"vgg16","128","3_6_7","658.21" +"inception3","128","3_6_7","727.47" +"alexnet","32","0_4_6","3295.75" +"resnet50","32","0_4_6","831.08" +"vgg16","32","0_4_6","484.66" +"inception3","32","0_4_6","580.10" +"alexnet","64","0_4_6","5809.04" +"resnet50","64","0_4_6","1006.98" +"vgg16","64","0_4_6","587.06" +"inception3","64","0_4_6","683.48" +"alexnet","128","0_4_6","9141.93" +"resnet50","128","0_4_6","1139.15" +"vgg16","128","0_4_6","648.12" +"inception3","128","0_4_6","742.75" +"alexnet","32","0_2_7","1806.62" +"resnet50","32","0_2_7","808.21" +"vgg16","32","0_2_7","326.40" +"inception3","32","0_2_7","567.02" +"alexnet","64","0_2_7","3417.95" +"resnet50","64","0_2_7","983.73" +"vgg16","64","0_2_7","376.15" +"inception3","64","0_2_7","668.89" +"alexnet","128","0_2_7","5782.30" +"resnet50","128","0_2_7","1107.51" +"vgg16","128","0_2_7","396.52" +"inception3","128","0_2_7","730.57" +"alexnet","32","0_1_4","2080.66" +"resnet50","32","0_1_4","834.74" +"vgg16","32","0_1_4","488.08" +"inception3","32","0_1_4","573.27" +"alexnet","64","0_1_4","3961.03" +"resnet50","64","0_1_4","1006.86" +"vgg16","64","0_1_4","588.83" +"inception3","64","0_1_4","667.29" +"alexnet","128","0_1_4","6563.84" +"resnet50","128","0_1_4","1139.71" +"vgg16","128","0_1_4","646.42" +"inception3","128","0_1_4","726.07" +"alexnet","32","2_4_6","2137.17" +"resnet50","32","2_4_6","847.69" +"vgg16","32","2_4_6","552.49" +"inception3","32","2_4_6","594.47" +"alexnet","64","2_4_6","4040.51" +"resnet50","64","2_4_6","1010.77" +"vgg16","64","2_4_6","631.46" +"inception3","64","2_4_6","691.96" +"alexnet","128","2_4_6","7225.70" +"resnet50","128","2_4_6","1151.76" +"vgg16","128","2_4_6","677.11" +"inception3","128","2_4_6","750.38" +"alexnet","32","0_4_7","3346.91" +"resnet50","32","0_4_7","834.23" +"vgg16","32","0_4_7","491.41" +"inception3","32","0_4_7","581.78" +"alexnet","64","0_4_7","5590.60" +"resnet50","64","0_4_7","1005.37" +"vgg16","64","0_4_7","586.35" +"inception3","64","0_4_7","684.98" +"alexnet","128","0_4_7","8399.14" +"resnet50","128","0_4_7","1129.51" +"vgg16","128","0_4_7","637.06" +"inception3","128","0_4_7","732.05" +"alexnet","32","0_1_2_3","4450.67" +"resnet50","32","0_1_2_3","1103.87" +"vgg16","32","0_1_2_3","741.17" +"inception3","32","0_1_2_3","759.75" +"alexnet","64","0_1_2_3","7978.19" +"resnet50","64","0_1_2_3","1310.09" +"vgg16","64","0_1_2_3","848.44" +"inception3","64","0_1_2_3","858.87" +"alexnet","128","0_1_2_3","11629.84" +"resnet50","128","0_1_2_3","1501.38" +"vgg16","128","0_1_2_3","908.14" +"inception3","128","0_1_2_3","971.64" +"alexnet","32","0_3_4_5","2356.73" +"resnet50","32","0_3_4_5","1046.15" +"vgg16","32","0_3_4_5","615.97" +"inception3","32","0_3_4_5","732.68" +"alexnet","64","0_3_4_5","4499.28" +"resnet50","64","0_3_4_5","1284.55" +"vgg16","64","0_3_4_5","763.83" +"inception3","64","0_3_4_5","868.29" +"alexnet","128","0_3_4_5","7917.65" +"resnet50","128","0_3_4_5","1489.12" +"vgg16","128","0_3_4_5","851.68" +"inception3","128","0_3_4_5","945.30" +"alexnet","32","0_2_4_7","2526.65" +"resnet50","32","0_2_4_7","1065.53" +"vgg16","32","0_2_4_7","635.49" +"inception3","32","0_2_4_7","750.58" +"alexnet","64","0_2_4_7","4827.72" +"resnet50","64","0_2_4_7","1263.11" +"vgg16","64","0_2_4_7","771.53" +"inception3","64","0_2_4_7","879.74" +"alexnet","128","0_2_4_7","8507.98" +"resnet50","128","0_2_4_7","1440.52" +"vgg16","128","0_2_4_7","851.54" +"inception3","128","0_2_4_7","956.98" +"alexnet","32","0_2_3_5","2154.12" +"resnet50","32","0_2_3_5","1037.28" +"vgg16","32","0_2_3_5","600.36" +"inception3","32","0_2_3_5","722.83" +"alexnet","64","0_2_3_5","4093.10" +"resnet50","64","0_2_3_5","1302.90" +"vgg16","64","0_2_3_5","741.78" +"inception3","64","0_2_3_5","852.84" +"alexnet","128","0_2_3_5","7121.02" +"resnet50","128","0_2_3_5","1485.92" +"vgg16","128","0_2_3_5","821.29" +"inception3","128","0_2_3_5","957.68" +"alexnet","32","0_1_2_4","2239.12" +"resnet50","32","0_1_2_4","1084.16" +"vgg16","32","0_1_2_4","577.55" +"inception3","32","0_1_2_4","735.85" +"alexnet","64","0_1_2_4","4239.91" +"resnet50","64","0_1_2_4","1317.19" +"vgg16","64","0_1_2_4","739.12" +"inception3","64","0_1_2_4","865.48" +"alexnet","128","0_1_2_4","7496.99" +"resnet50","128","0_1_2_4","1471.40" +"vgg16","128","0_1_2_4","835.41" +"inception3","128","0_1_2_4","953.25" +"alexnet","32","3_5_6_7","2270.18" +"resnet50","32","3_5_6_7","1074.56" +"vgg16","32","3_5_6_7","619.66" +"inception3","32","3_5_6_7","745.42" +"alexnet","64","3_5_6_7","4304.71" +"resnet50","64","3_5_6_7","1303.14" +"vgg16","64","3_5_6_7","755.56" +"inception3","64","3_5_6_7","872.17" +"alexnet","128","3_5_6_7","7515.23" +"resnet50","128","3_5_6_7","1479.46" +"vgg16","128","3_5_6_7","837.28" +"inception3","128","3_5_6_7","969.23" +"alexnet","32","0_3_4_6","2403.86" +"resnet50","32","0_3_4_6","1066.05" +"vgg16","32","0_3_4_6","617.40" +"inception3","32","0_3_4_6","751.26" +"alexnet","64","0_3_4_6","4605.30" +"resnet50","64","0_3_4_6","1310.08" +"vgg16","64","0_3_4_6","767.08" +"inception3","64","0_3_4_6","871.99" +"alexnet","128","0_3_4_6","8100.43" +"resnet50","128","0_3_4_6","1504.77" +"vgg16","128","0_3_4_6","852.86" +"inception3","128","0_3_4_6","956.24" +"alexnet","32","1_2_3_6","2442.09" +"resnet50","32","1_2_3_6","1060.13" +"vgg16","32","1_2_3_6","594.30" +"inception3","32","1_2_3_6","729.95" +"alexnet","64","1_2_3_6","4632.19" +"resnet50","64","1_2_3_6","1310.13" +"vgg16","64","1_2_3_6","739.89" +"inception3","64","1_2_3_6","876.71" +"alexnet","128","1_2_3_6","7778.67" +"resnet50","128","1_2_3_6","1500.83" +"vgg16","128","1_2_3_6","841.58" +"inception3","128","1_2_3_6","947.68" +"alexnet","32","0_1_4_7","2478.37" +"resnet50","32","0_1_4_7","1065.05" +"vgg16","32","0_1_4_7","636.10" +"inception3","32","0_1_4_7","736.89" +"alexnet","64","0_1_4_7","4722.28" +"resnet50","64","0_1_4_7","1298.48" +"vgg16","64","0_1_4_7","771.65" +"inception3","64","0_1_4_7","876.84" +"alexnet","128","0_1_4_7","8167.83" +"resnet50","128","0_1_4_7","1484.56" +"vgg16","128","0_1_4_7","845.65" +"inception3","128","0_1_4_7","931.71" +"alexnet","32","0_3_5_6","1737.61" +"resnet50","32","0_3_5_6","1064.76" +"vgg16","32","0_3_5_6","481.60" +"inception3","32","0_3_5_6","747.24" +"alexnet","64","0_3_5_6","3347.94" +"resnet50","64","0_3_5_6","1311.45" +"vgg16","64","0_3_5_6","635.82" +"inception3","64","0_3_5_6","898.57" +"alexnet","128","0_3_5_6","6270.89" +"resnet50","128","0_3_5_6","1476.25" +"vgg16","128","0_3_5_6","758.99" +"inception3","128","0_3_5_6","952.43" +"alexnet","32","2_3_5_7","1700.29" +"resnet50","32","2_3_5_7","1029.67" +"vgg16","32","2_3_5_7","466.90" +"inception3","32","2_3_5_7","745.50" +"alexnet","64","2_3_5_7","3293.33" +"resnet50","64","2_3_5_7","1285.94" +"vgg16","64","2_3_5_7","636.26" +"inception3","64","2_3_5_7","886.18" +"alexnet","128","2_3_5_7","6146.11" +"resnet50","128","2_3_5_7","1493.41" +"vgg16","128","2_3_5_7","716.71" +"inception3","128","2_3_5_7","979.33" +"alexnet","32","0_4_5_7","2488.34" +"resnet50","32","0_4_5_7","1094.58" +"vgg16","32","0_4_5_7","603.17" +"inception3","32","0_4_5_7","741.65" +"alexnet","64","0_4_5_7","4807.45" +"resnet50","64","0_4_5_7","1307.96" +"vgg16","64","0_4_5_7","755.99" +"inception3","64","0_4_5_7","877.22" +"alexnet","128","0_4_5_7","8182.41" +"resnet50","128","0_4_5_7","1421.20" +"vgg16","128","0_4_5_7","840.83" +"inception3","128","0_4_5_7","949.30" +"alexnet","32","1_3_6_7","1763.40" +"resnet50","32","1_3_6_7","1033.16" +"vgg16","32","1_3_6_7","482.33" +"inception3","32","1_3_6_7","737.95" +"alexnet","64","1_3_6_7","3397.71" +"resnet50","64","1_3_6_7","1303.97" +"vgg16","64","1_3_6_7","637.57" +"inception3","64","1_3_6_7","880.25" +"alexnet","128","1_3_6_7","6204.49" +"resnet50","128","1_3_6_7","1464.87" +"vgg16","128","1_3_6_7","732.30" +"inception3","128","1_3_6_7","958.16" +"alexnet","32","0_1_3_4","2187.20" +"resnet50","32","0_1_3_4","1089.29" +"vgg16","32","0_1_3_4","583.58" +"inception3","32","0_1_3_4","736.17" +"alexnet","64","0_1_3_4","4198.57" +"resnet50","64","0_1_3_4","1301.52" +"vgg16","64","0_1_3_4","736.13" +"inception3","64","0_1_3_4","869.85" +"alexnet","128","0_1_3_4","7392.07" +"resnet50","128","0_1_3_4","1485.39" +"vgg16","128","0_1_3_4","829.78" +"inception3","128","0_1_3_4","950.88" +"alexnet","32","0_2_6_7","2499.71" +"resnet50","32","0_2_6_7","1044.36" +"vgg16","32","0_2_6_7","615.11" +"inception3","32","0_2_6_7","729.64" +"alexnet","64","0_2_6_7","4795.19" +"resnet50","64","0_2_6_7","1262.72" +"vgg16","64","0_2_6_7","758.15" +"inception3","64","0_2_6_7","872.71" +"alexnet","128","0_2_6_7","8285.78" +"resnet50","128","0_2_6_7","1473.78" +"vgg16","128","0_2_6_7","843.84" +"inception3","128","0_2_6_7","963.80" +"alexnet","32","0_1_3_7","3349.51" +"resnet50","32","0_1_3_7","1080.61" +"vgg16","32","0_1_3_7","692.24" +"inception3","32","0_1_3_7","728.45" +"alexnet","64","0_1_3_7","6260.71" +"resnet50","64","0_1_3_7","1312.11" +"vgg16","64","0_1_3_7","805.54" +"inception3","64","0_1_3_7","868.18" +"alexnet","128","0_1_3_7","9879.27" +"resnet50","128","0_1_3_7","1475.91" +"vgg16","128","0_1_3_7","871.76" +"inception3","128","0_1_3_7","949.59" +"alexnet","32","1_2_4_6","1718.99" +"resnet50","32","1_2_4_6","1081.31" +"vgg16","32","1_2_4_6","471.40" +"inception3","32","1_2_4_6","763.19" +"alexnet","64","1_2_4_6","3334.32" +"resnet50","64","1_2_4_6","1253.44" +"vgg16","64","1_2_4_6","630.73" +"inception3","64","1_2_4_6","901.00" +"alexnet","128","1_2_4_6","6166.72" +"resnet50","128","1_2_4_6","1495.90" +"vgg16","128","1_2_4_6","736.72" +"inception3","128","1_2_4_6","972.86" +"alexnet","32","2_3_4_7","1775.71" +"resnet50","32","2_3_4_7","1068.47" +"vgg16","32","2_3_4_7","485.57" +"inception3","32","2_3_4_7","753.00" +"alexnet","64","2_3_4_7","3446.31" +"resnet50","64","2_3_4_7","1296.30" +"vgg16","64","2_3_4_7","633.70" +"inception3","64","2_3_4_7","876.59" +"alexnet","128","2_3_4_7","6265.96" +"resnet50","128","2_3_4_7","1410.33" +"vgg16","128","2_3_4_7","734.09" +"inception3","128","2_3_4_7","966.52" +"alexnet","32","4_5_6_7","4481.74" +"resnet50","32","4_5_6_7","1109.83" +"vgg16","32","4_5_6_7","740.63" +"inception3","32","4_5_6_7","759.92" +"alexnet","64","4_5_6_7","8018.66" +"resnet50","64","4_5_6_7","1250.34" +"vgg16","64","4_5_6_7","836.66" +"inception3","64","4_5_6_7","884.37" +"alexnet","128","4_5_6_7","11602.69" +"resnet50","128","4_5_6_7","1402.23" +"vgg16","128","4_5_6_7","893.52" +"inception3","128","4_5_6_7","956.86" +"alexnet","32","1_2_5_6","2793.13" +"resnet50","32","1_2_5_6","1096.67" +"vgg16","32","1_2_5_6","639.02" +"inception3","32","1_2_5_6","768.34" +"alexnet","64","1_2_5_6","5397.48" +"resnet50","64","1_2_5_6","1332.06" +"vgg16","64","1_2_5_6","780.68" +"inception3","64","1_2_5_6","884.51" +"alexnet","128","1_2_5_6","8853.75" +"resnet50","128","1_2_5_6","1487.90" +"vgg16","128","1_2_5_6","867.52" +"inception3","128","1_2_5_6","971.65" +"alexnet","32","0_2_5_6","1763.37" +"resnet50","32","0_2_5_6","1075.75" +"vgg16","32","0_2_5_6","489.44" +"inception3","32","0_2_5_6","754.01" +"alexnet","64","0_2_5_6","3438.45" +"resnet50","64","0_2_5_6","1310.64" +"vgg16","64","0_2_5_6","640.33" +"inception3","64","0_2_5_6","897.38" +"alexnet","128","0_2_5_6","6342.04" +"resnet50","128","0_2_5_6","1503.36" +"vgg16","128","0_2_5_6","765.27" +"inception3","128","0_2_5_6","968.45" +"alexnet","32","2_3_6_7","2604.63" +"resnet50","32","2_3_6_7","1059.00" +"vgg16","32","2_3_6_7","610.98" +"inception3","32","2_3_6_7","744.27" +"alexnet","64","2_3_6_7","4986.23" +"resnet50","64","2_3_6_7","1291.03" +"vgg16","64","2_3_6_7","754.78" +"inception3","64","2_3_6_7","879.34" +"alexnet","128","2_3_6_7","8477.90" +"resnet50","128","2_3_6_7","1479.41" +"vgg16","128","2_3_6_7","844.95" +"inception3","128","2_3_6_7","941.62" +"alexnet","32","0_5_6_7","2193.29" +"resnet50","32","0_5_6_7","1049.74" +"vgg16","32","0_5_6_7","611.72" +"inception3","32","0_5_6_7","728.77" +"alexnet","64","0_5_6_7","4177.98" +"resnet50","64","0_5_6_7","1272.70" +"vgg16","64","0_5_6_7","722.99" +"inception3","64","0_5_6_7","867.05" +"alexnet","128","0_5_6_7","7393.27" +"resnet50","128","0_5_6_7","1450.39" +"vgg16","128","0_5_6_7","776.16" +"inception3","128","0_5_6_7","964.02" +"alexnet","32","0_2_4_6","2685.07" +"resnet50","32","0_2_4_6","1084.62" +"vgg16","32","0_2_4_6","629.81" +"inception3","32","0_2_4_6","756.66" +"alexnet","64","0_2_4_6","5171.91" +"resnet50","64","0_2_4_6","1298.90" +"vgg16","64","0_2_4_6","765.82" +"inception3","64","0_2_4_6","895.33" +"alexnet","128","0_2_4_6","8714.14" +"resnet50","128","0_2_4_6","1498.72" +"vgg16","128","0_2_4_6","854.70" +"inception3","128","0_2_4_6","953.60" +"alexnet","32","0_2_3_6","2566.31" +"resnet50","32","0_2_3_6","1073.37" +"vgg16","32","0_2_3_6","608.60" +"inception3","32","0_2_3_6","733.45" +"alexnet","64","0_2_3_6","4792.27" +"resnet50","64","0_2_3_6","1299.55" +"vgg16","64","0_2_3_6","758.43" +"inception3","64","0_2_3_6","862.73" +"alexnet","128","0_2_3_6","8190.38" +"resnet50","128","0_2_3_6","1418.46" +"vgg16","128","0_2_3_6","847.73" +"inception3","128","0_2_3_6","957.75" +"alexnet","32","0_1_2_5","2506.08" +"resnet50","32","0_1_2_5","1074.94" +"vgg16","32","0_1_2_5","596.66" +"inception3","32","0_1_2_5","739.29" +"alexnet","64","0_1_2_5","4602.09" +"resnet50","64","0_1_2_5","1289.85" +"vgg16","64","0_1_2_5","743.38" +"inception3","64","0_1_2_5","878.91" +"alexnet","128","0_1_2_5","7874.66" +"resnet50","128","0_1_2_5","1491.51" +"vgg16","128","0_1_2_5","839.72" +"inception3","128","0_1_2_5","954.27" +"alexnet","32","0_3_4_7","2846.93" +"resnet50","32","0_3_4_7","1086.71" +"vgg16","32","0_3_4_7","637.60" +"inception3","32","0_3_4_7","766.88" +"alexnet","64","0_3_4_7","5427.18" +"resnet50","64","0_3_4_7","1307.90" +"vgg16","64","0_3_4_7","780.21" +"inception3","64","0_3_4_7","890.91" +"alexnet","128","0_3_4_7","8898.28" +"resnet50","128","0_3_4_7","1470.08" +"vgg16","128","0_3_4_7","854.73" +"inception3","128","0_3_4_7","955.92" +"alexnet","32","1_2_3_5","2261.27" +"resnet50","32","1_2_3_5","1078.87" +"vgg16","32","1_2_3_5","584.85" +"inception3","32","1_2_3_5","738.49" +"alexnet","64","1_2_3_5","4293.10" +"resnet50","64","1_2_3_5","1295.63" +"vgg16","64","1_2_3_5","735.69" +"inception3","64","1_2_3_5","885.13" +"alexnet","128","1_2_3_5","7517.91" +"resnet50","128","1_2_3_5","1506.67" +"vgg16","128","1_2_3_5","838.61" +"inception3","128","1_2_3_5","965.83" +"alexnet","32","0_1_2_6","3391.07" +"resnet50","32","0_1_2_6","1082.50" +"vgg16","32","0_1_2_6","696.72" +"inception3","32","0_1_2_6","739.59" +"alexnet","64","0_1_2_6","6253.16" +"resnet50","64","0_1_2_6","1273.06" +"vgg16","64","0_1_2_6","817.44" +"inception3","64","0_1_2_6","863.48" +"alexnet","128","0_1_2_6","9807.84" +"resnet50","128","0_1_2_6","1487.96" +"vgg16","128","0_1_2_6","887.33" +"inception3","128","0_1_2_6","964.80" +"alexnet","32","1_3_4_7","1766.26" +"resnet50","32","1_3_4_7","1073.44" +"vgg16","32","1_3_4_7","499.08" +"inception3","32","1_3_4_7","737.80" +"alexnet","64","1_3_4_7","3455.93" +"resnet50","64","1_3_4_7","1298.77" +"vgg16","64","1_3_4_7","629.65" +"inception3","64","1_3_4_7","885.72" +"alexnet","128","1_3_4_7","6340.51" +"resnet50","128","1_3_4_7","1471.51" +"vgg16","128","1_3_4_7","749.73" +"inception3","128","1_3_4_7","963.40" +"alexnet","32","1_4_5_6","3914.54" +"resnet50","32","1_4_5_6","1086.15" +"vgg16","32","1_4_5_6","724.87" +"inception3","32","1_4_5_6","739.18" +"alexnet","64","1_4_5_6","7366.38" +"resnet50","64","1_4_5_6","1262.19" +"vgg16","64","1_4_5_6","833.55" +"inception3","64","1_4_5_6","875.90" +"alexnet","128","1_4_5_6","11518.18" +"resnet50","128","1_4_5_6","1503.92" +"vgg16","128","1_4_5_6","891.92" +"inception3","128","1_4_5_6","969.38" +"alexnet","32","3_4_5_7","2100.80" +"resnet50","32","3_4_5_7","1060.81" +"vgg16","32","3_4_5_7","579.13" +"inception3","32","3_4_5_7","749.35" +"alexnet","64","3_4_5_7","4042.81" +"resnet50","64","3_4_5_7","1325.45" +"vgg16","64","3_4_5_7","724.91" +"inception3","64","3_4_5_7","888.85" +"alexnet","128","3_4_5_7","7046.58" +"resnet50","128","3_4_5_7","1479.38" +"vgg16","128","3_4_5_7","818.52" +"inception3","128","3_4_5_7","965.26" +"alexnet","32","0_1_4_5","2597.13" +"resnet50","32","0_1_4_5","1068.01" +"vgg16","32","0_1_4_5","611.08" +"inception3","32","0_1_4_5","744.05" +"alexnet","64","0_1_4_5","4983.33" +"resnet50","64","0_1_4_5","1287.35" +"vgg16","64","0_1_4_5","759.78" +"inception3","64","0_1_4_5","881.03" +"alexnet","128","0_1_4_5","8406.38" +"resnet50","128","0_1_4_5","1492.98" +"vgg16","128","0_1_4_5","848.03" +"inception3","128","0_1_4_5","958.66" +"alexnet","32","1_5_6_7","2810.66" +"resnet50","32","1_5_6_7","1061.93" +"vgg16","32","1_5_6_7","644.96" +"inception3","32","1_5_6_7","741.90" +"alexnet","64","1_5_6_7","5355.63" +"resnet50","64","1_5_6_7","1262.22" +"vgg16","64","1_5_6_7","781.11" +"inception3","64","1_5_6_7","883.76" +"alexnet","128","1_5_6_7","8978.15" +"resnet50","128","1_5_6_7","1489.93" +"vgg16","128","1_5_6_7","850.34" +"inception3","128","1_5_6_7","959.79" +"alexnet","32","0_1_5_7","2502.73" +"resnet50","32","0_1_5_7","1039.61" +"vgg16","32","0_1_5_7","615.96" +"inception3","32","0_1_5_7","741.89" +"alexnet","64","0_1_5_7","4828.09" +"resnet50","64","0_1_5_7","1280.11" +"vgg16","64","0_1_5_7","756.82" +"inception3","64","0_1_5_7","864.56" +"alexnet","128","0_1_5_7","8395.78" +"resnet50","128","0_1_5_7","1450.85" +"vgg16","128","0_1_5_7","838.55" +"inception3","128","0_1_5_7","948.52" +"alexnet","32","0_4_6_7","2658.20" +"resnet50","32","0_4_6_7","1064.83" +"vgg16","32","0_4_6_7","623.62" +"inception3","32","0_4_6_7","734.78" +"alexnet","64","0_4_6_7","5079.85" +"resnet50","64","0_4_6_7","1288.36" +"vgg16","64","0_4_6_7","760.56" +"inception3","64","0_4_6_7","879.03" +"alexnet","128","0_4_6_7","8573.14" +"resnet50","128","0_4_6_7","1460.30" +"vgg16","128","0_4_6_7","848.16" +"inception3","128","0_4_6_7","953.52" +"alexnet","32","1_4_5_7","3638.31" +"resnet50","32","1_4_5_7","1060.29" +"vgg16","32","1_4_5_7","688.71" +"inception3","32","1_4_5_7","731.76" +"alexnet","64","1_4_5_7","6753.55" +"resnet50","64","1_4_5_7","1316.53" +"vgg16","64","1_4_5_7","807.94" +"inception3","64","1_4_5_7","881.22" +"alexnet","128","1_4_5_7","10429.91" +"resnet50","128","1_4_5_7","1461.66" +"vgg16","128","1_4_5_7","872.47" +"inception3","128","1_4_5_7","962.62" +"alexnet","32","1_3_5_6","2521.01" +"resnet50","32","1_3_5_6","1072.68" +"vgg16","32","1_3_5_6","637.64" +"inception3","32","1_3_5_6","742.43" +"alexnet","64","1_3_5_6","4803.78" +"resnet50","64","1_3_5_6","1320.85" +"vgg16","64","1_3_5_6","776.34" +"inception3","64","1_3_5_6","878.24" +"alexnet","128","1_3_5_6","8425.15" +"resnet50","128","1_3_5_6","1491.99" +"vgg16","128","1_3_5_6","866.73" +"inception3","128","1_3_5_6","967.21" +"alexnet","32","1_2_4_7","1726.44" +"resnet50","32","1_2_4_7","1052.43" +"vgg16","32","1_2_4_7","488.92" +"inception3","32","1_2_4_7","749.79" +"alexnet","64","1_2_4_7","3385.31" +"resnet50","64","1_2_4_7","1296.25" +"vgg16","64","1_2_4_7","650.60" +"inception3","64","1_2_4_7","887.70" +"alexnet","128","1_2_4_7","6203.39" +"resnet50","128","1_2_4_7","1472.73" +"vgg16","128","1_2_4_7","771.90" +"inception3","128","1_2_4_7","954.41" +"alexnet","32","3_4_5_6","2087.69" +"resnet50","32","3_4_5_6","1063.89" +"vgg16","32","3_4_5_6","596.13" +"inception3","32","3_4_5_6","740.96" +"alexnet","64","3_4_5_6","3973.82" +"resnet50","64","3_4_5_6","1291.14" +"vgg16","64","3_4_5_6","718.05" +"inception3","64","3_4_5_6","885.21" +"alexnet","128","3_4_5_6","7098.89" +"resnet50","128","3_4_5_6","1412.25" +"vgg16","128","3_4_5_6","809.38" +"inception3","128","3_4_5_6","964.17" +"alexnet","32","1_3_4_6","1642.05" +"resnet50","32","1_3_4_6","1035.06" +"vgg16","32","1_3_4_6","465.32" +"inception3","32","1_3_4_6","746.58" +"alexnet","64","1_3_4_6","3254.44" +"resnet50","64","1_3_4_6","1294.26" +"vgg16","64","1_3_4_6","633.94" +"inception3","64","1_3_4_6","892.70" +"alexnet","128","1_3_4_6","5936.70" +"resnet50","128","1_3_4_6","1486.92" +"vgg16","128","1_3_4_6","729.36" +"inception3","128","1_3_4_6","958.15" +"alexnet","32","0_1_3_6","2063.19" +"resnet50","32","0_1_3_6","1048.94" +"vgg16","32","0_1_3_6","579.06" +"inception3","32","0_1_3_6","721.00" +"alexnet","64","0_1_3_6","3954.46" +"resnet50","64","0_1_3_6","1297.66" +"vgg16","64","0_1_3_6","715.97" +"inception3","64","0_1_3_6","871.11" +"alexnet","128","0_1_3_6","6961.20" +"resnet50","128","0_1_3_6","1485.55" +"vgg16","128","0_1_3_6","787.32" +"inception3","128","0_1_3_6","920.77" +"alexnet","32","2_3_4_5","1606.35" +"resnet50","32","2_3_4_5","1017.93" +"vgg16","32","2_3_4_5","459.36" +"inception3","32","2_3_4_5","719.65" +"alexnet","64","2_3_4_5","3152.64" +"resnet50","64","2_3_4_5","1278.63" +"vgg16","64","2_3_4_5","616.14" +"inception3","64","2_3_4_5","874.68" +"alexnet","128","2_3_4_5","5770.13" +"resnet50","128","2_3_4_5","1411.25" +"vgg16","128","2_3_4_5","751.78" +"inception3","128","2_3_4_5","981.17" +"alexnet","32","0_3_6_7","1774.75" +"resnet50","32","0_3_6_7","1061.81" +"vgg16","32","0_3_6_7","483.00" +"inception3","32","0_3_6_7","728.37" +"alexnet","64","0_3_6_7","3439.74" +"resnet50","64","0_3_6_7","1299.60" +"vgg16","64","0_3_6_7","620.16" +"inception3","64","0_3_6_7","876.27" +"alexnet","128","0_3_6_7","6187.32" +"resnet50","128","0_3_6_7","1452.60" +"vgg16","128","0_3_6_7","739.09" +"inception3","128","0_3_6_7","956.54" +"alexnet","32","1_3_4_5","1648.82" +"resnet50","32","1_3_4_5","1053.53" +"vgg16","32","1_3_4_5","474.88" +"inception3","32","1_3_4_5","735.47" +"alexnet","64","1_3_4_5","3223.67" +"resnet50","64","1_3_4_5","1277.48" +"vgg16","64","1_3_4_5","631.37" +"inception3","64","1_3_4_5","878.41" +"alexnet","128","1_3_4_5","5872.95" +"resnet50","128","1_3_4_5","1478.75" +"vgg16","128","1_3_4_5","757.55" +"inception3","128","1_3_4_5","974.27" +"alexnet","32","3_4_6_7","2198.00" +"resnet50","32","3_4_6_7","1086.50" +"vgg16","32","3_4_6_7","605.22" +"inception3","32","3_4_6_7","740.45" +"alexnet","64","3_4_6_7","4182.17" +"resnet50","64","3_4_6_7","1300.90" +"vgg16","64","3_4_6_7","741.63" +"inception3","64","3_4_6_7","883.86" +"alexnet","128","3_4_6_7","7431.97" +"resnet50","128","3_4_6_7","1471.75" +"vgg16","128","3_4_6_7","828.85" +"inception3","128","3_4_6_7","957.48" +"alexnet","32","2_5_6_7","3848.35" +"resnet50","32","2_5_6_7","1064.28" +"vgg16","32","2_5_6_7","725.85" +"inception3","32","2_5_6_7","743.96" +"alexnet","64","2_5_6_7","7084.97" +"resnet50","64","2_5_6_7","1307.24" +"vgg16","64","2_5_6_7","834.47" +"inception3","64","2_5_6_7","870.81" +"alexnet","128","2_5_6_7","11295.07" +"resnet50","128","2_5_6_7","1398.24" +"vgg16","128","2_5_6_7","887.01" +"inception3","128","2_5_6_7","957.30" +"alexnet","32","0_2_5_7","1635.41" +"resnet50","32","0_2_5_7","1055.00" +"vgg16","32","0_2_5_7","462.32" +"inception3","32","0_2_5_7","753.23" +"alexnet","64","0_2_5_7","3200.73" +"resnet50","64","0_2_5_7","1300.62" +"vgg16","64","0_2_5_7","629.43" +"inception3","64","0_2_5_7","889.05" +"alexnet","128","0_2_5_7","5964.19" +"resnet50","128","0_2_5_7","1483.04" +"vgg16","128","0_2_5_7","741.28" +"inception3","128","0_2_5_7","960.95" +"alexnet","32","2_4_5_6","2171.39" +"resnet50","32","2_4_5_6","1070.82" +"vgg16","32","2_4_5_6","601.90" +"inception3","32","2_4_5_6","746.16" +"alexnet","64","2_4_5_6","4150.33" +"resnet50","64","2_4_5_6","1243.55" +"vgg16","64","2_4_5_6","742.72" +"inception3","64","2_4_5_6","896.52" +"alexnet","128","2_4_5_6","7143.63" +"resnet50","128","2_4_5_6","1401.09" +"vgg16","128","2_4_5_6","839.61" +"inception3","128","2_4_5_6","958.62" +"alexnet","32","0_2_4_5","2354.15" +"resnet50","32","0_2_4_5","1032.89" +"vgg16","32","0_2_4_5","613.87" +"inception3","32","0_2_4_5","729.32" +"alexnet","64","0_2_4_5","4495.06" +"resnet50","64","0_2_4_5","1260.24" +"vgg16","64","0_2_4_5","765.78" +"inception3","64","0_2_4_5","876.14" +"alexnet","128","0_2_4_5","7904.20" +"resnet50","128","0_2_4_5","1492.16" +"vgg16","128","0_2_4_5","857.77" +"inception3","128","0_2_4_5","945.31" +"alexnet","32","0_2_3_7","3579.71" +"resnet50","32","0_2_3_7","1074.78" +"vgg16","32","0_2_3_7","717.17" +"inception3","32","0_2_3_7","732.96" +"alexnet","64","0_2_3_7","6539.66" +"resnet50","64","0_2_3_7","1289.01" +"vgg16","64","0_2_3_7","827.88" +"inception3","64","0_2_3_7","880.23" +"alexnet","128","0_2_3_7","10589.38" +"resnet50","128","0_2_3_7","1496.59" +"vgg16","128","0_2_3_7","892.50" +"inception3","128","0_2_3_7","971.33" +"alexnet","32","1_2_6_7","2550.26" +"resnet50","32","1_2_6_7","1036.08" +"vgg16","32","1_2_6_7","611.47" +"inception3","32","1_2_6_7","725.47" +"alexnet","64","1_2_6_7","4903.59" +"resnet50","64","1_2_6_7","1244.58" +"vgg16","64","1_2_6_7","756.48" +"inception3","64","1_2_6_7","863.78" +"alexnet","128","1_2_6_7","8454.19" +"resnet50","128","1_2_6_7","1476.63" +"vgg16","128","1_2_6_7","847.58" +"inception3","128","1_2_6_7","948.30" +"alexnet","32","1_2_3_4","2101.41" +"resnet50","32","1_2_3_4","1052.81" +"vgg16","32","1_2_3_4","578.65" +"inception3","32","1_2_3_4","719.45" +"alexnet","64","1_2_3_4","4015.34" +"resnet50","64","1_2_3_4","1287.58" +"vgg16","64","1_2_3_4","717.66" +"inception3","64","1_2_3_4","863.78" +"alexnet","128","1_2_3_4","7011.33" +"resnet50","128","1_2_3_4","1436.30" +"vgg16","128","1_2_3_4","811.12" +"inception3","128","1_2_3_4","934.75" +"alexnet","32","0_2_3_4","2319.82" +"resnet50","32","0_2_3_4","1057.44" +"vgg16","32","0_2_3_4","601.04" +"inception3","32","0_2_3_4","743.73" +"alexnet","64","0_2_3_4","4336.51" +"resnet50","64","0_2_3_4","1310.97" +"vgg16","64","0_2_3_4","751.55" +"inception3","64","0_2_3_4","866.25" +"alexnet","128","0_2_3_4","7529.48" +"resnet50","128","0_2_3_4","1507.12" +"vgg16","128","0_2_3_4","844.34" +"inception3","128","0_2_3_4","950.39" +"alexnet","32","0_1_2_7","2103.28" +"resnet50","32","0_1_2_7","1043.94" +"vgg16","32","0_1_2_7","583.05" +"inception3","32","0_1_2_7","711.30" +"alexnet","64","0_1_2_7","4000.02" +"resnet50","64","0_1_2_7","1281.76" +"vgg16","64","0_1_2_7","719.24" +"inception3","64","0_1_2_7","848.68" +"alexnet","128","0_1_2_7","6919.72" +"resnet50","128","0_1_2_7","1474.99" +"vgg16","128","0_1_2_7","803.57" +"inception3","128","0_1_2_7","947.26" +"alexnet","32","1_2_3_7","3405.12" +"resnet50","32","1_2_3_7","1083.20" +"vgg16","32","1_2_3_7","687.88" +"inception3","32","1_2_3_7","725.02" +"alexnet","64","1_2_3_7","6251.55" +"resnet50","64","1_2_3_7","1305.84" +"vgg16","64","1_2_3_7","808.06" +"inception3","64","1_2_3_7","856.85" +"alexnet","128","1_2_3_7","10091.66" +"resnet50","128","1_2_3_7","1469.37" +"vgg16","128","1_2_3_7","879.80" +"inception3","128","1_2_3_7","950.54" +"alexnet","32","0_1_4_6","2334.40" +"resnet50","32","0_1_4_6","1057.47" +"vgg16","32","0_1_4_6","616.21" +"inception3","32","0_1_4_6","736.38" +"alexnet","64","0_1_4_6","4448.70" +"resnet50","64","0_1_4_6","1260.04" +"vgg16","64","0_1_4_6","761.15" +"inception3","64","0_1_4_6","872.99" +"alexnet","128","0_1_4_6","7732.95" +"resnet50","128","0_1_4_6","1479.43" +"vgg16","128","0_1_4_6","844.91" +"inception3","128","0_1_4_6","955.88" +"alexnet","32","0_3_5_7","1726.13" +"resnet50","32","0_3_5_7","1074.97" +"vgg16","32","0_3_5_7","474.72" +"inception3","32","0_3_5_7","754.76" +"alexnet","64","0_3_5_7","3343.54" +"resnet50","64","0_3_5_7","1247.79" +"vgg16","64","0_3_5_7","632.21" +"inception3","64","0_3_5_7","894.63" +"alexnet","128","0_3_5_7","6162.82" +"resnet50","128","0_3_5_7","1492.81" +"vgg16","128","0_3_5_7","725.71" +"inception3","128","0_3_5_7","947.66" +"alexnet","32","0_1_5_6","2624.90" +"resnet50","32","0_1_5_6","1059.91" +"vgg16","32","0_1_5_6","638.85" +"inception3","32","0_1_5_6","736.08" +"alexnet","64","0_1_5_6","5040.98" +"resnet50","64","0_1_5_6","1240.29" +"vgg16","64","0_1_5_6","778.13" +"inception3","64","0_1_5_6","860.36" +"alexnet","128","0_1_5_6","8818.97" +"resnet50","128","0_1_5_6","1422.96" +"vgg16","128","0_1_5_6","853.59" +"inception3","128","0_1_5_6","936.50" +"alexnet","32","1_3_5_7","2683.02" +"resnet50","32","1_3_5_7","1096.19" +"vgg16","32","1_3_5_7","623.13" +"inception3","32","1_3_5_7","756.16" +"alexnet","64","1_3_5_7","5108.15" +"resnet50","64","1_3_5_7","1315.23" +"vgg16","64","1_3_5_7","759.67" +"inception3","64","1_3_5_7","877.68" +"alexnet","128","1_3_5_7","8696.88" +"resnet50","128","1_3_5_7","1477.01" +"vgg16","128","1_3_5_7","848.06" +"inception3","128","1_3_5_7","964.49" +"alexnet","32","0_4_5_6","2643.23" +"resnet50","32","0_4_5_6","1055.51" +"vgg16","32","0_4_5_6","628.09" +"inception3","32","0_4_5_6","734.57" +"alexnet","64","0_4_5_6","5098.51" +"resnet50","64","0_4_5_6","1325.65" +"vgg16","64","0_4_5_6","772.31" +"inception3","64","0_4_5_6","866.67" +"alexnet","128","0_4_5_6","8655.70" +"resnet50","128","0_4_5_6","1413.59" +"vgg16","128","0_4_5_6","859.95" +"inception3","128","0_4_5_6","972.33" +"alexnet","32","2_4_6_7","3510.08" +"resnet50","32","2_4_6_7","1067.53" +"vgg16","32","2_4_6_7","700.53" +"inception3","32","2_4_6_7","742.03" +"alexnet","64","2_4_6_7","6411.75" +"resnet50","64","2_4_6_7","1294.08" +"vgg16","64","2_4_6_7","815.06" +"inception3","64","2_4_6_7","867.03" +"alexnet","128","2_4_6_7","10336.64" +"resnet50","128","2_4_6_7","1488.99" +"vgg16","128","2_4_6_7","873.89" +"inception3","128","2_4_6_7","955.98" +"alexnet","32","0_1_6_7","1653.86" +"resnet50","32","0_1_6_7","1041.01" +"vgg16","32","0_1_6_7","474.83" +"inception3","32","0_1_6_7","723.42" +"alexnet","64","0_1_6_7","3218.06" +"resnet50","64","0_1_6_7","1264.94" +"vgg16","64","0_1_6_7","630.80" +"inception3","64","0_1_6_7","870.68" +"alexnet","128","0_1_6_7","5860.27" +"resnet50","128","0_1_6_7","1462.48" +"vgg16","128","0_1_6_7","720.57" +"inception3","128","0_1_6_7","949.11" +"alexnet","32","0_1_3_5","2484.45" +"resnet50","32","0_1_3_5","1073.36" +"vgg16","32","0_1_3_5","594.68" +"inception3","32","0_1_3_5","742.14" +"alexnet","64","0_1_3_5","4642.74" +"resnet50","64","0_1_3_5","1302.13" +"vgg16","64","0_1_3_5","744.95" +"inception3","64","0_1_3_5","876.80" +"alexnet","128","0_1_3_5","7997.45" +"resnet50","128","0_1_3_5","1476.91" +"vgg16","128","0_1_3_5","836.07" +"inception3","128","0_1_3_5","946.61" +"alexnet","32","2_4_5_7","2038.98" +"resnet50","32","2_4_5_7","1041.34" +"vgg16","32","2_4_5_7","570.29" +"inception3","32","2_4_5_7","729.67" +"alexnet","64","2_4_5_7","3893.32" +"resnet50","64","2_4_5_7","1283.24" +"vgg16","64","2_4_5_7","689.05" +"inception3","64","2_4_5_7","882.87" +"alexnet","128","2_4_5_7","6910.99" +"resnet50","128","2_4_5_7","1493.99" +"vgg16","128","2_4_5_7","779.97" +"inception3","128","2_4_5_7","971.35" +"alexnet","32","1_4_6_7","2133.74" +"resnet50","32","1_4_6_7","1065.26" +"vgg16","32","1_4_6_7","581.22" +"inception3","32","1_4_6_7","730.57" +"alexnet","64","1_4_6_7","4053.08" +"resnet50","64","1_4_6_7","1245.20" +"vgg16","64","1_4_6_7","700.96" +"inception3","64","1_4_6_7","880.38" +"alexnet","128","1_4_6_7","7150.40" +"resnet50","128","1_4_6_7","1460.20" +"vgg16","128","1_4_6_7","771.73" +"inception3","128","1_4_6_7","966.67" +"alexnet","32","1_2_4_5","1659.42" +"resnet50","32","1_2_4_5","1068.29" +"vgg16","32","1_2_4_5","470.06" +"inception3","32","1_2_4_5","726.71" +"alexnet","64","1_2_4_5","3233.41" +"resnet50","64","1_2_4_5","1301.97" +"vgg16","64","1_2_4_5","646.86" +"inception3","64","1_2_4_5","861.89" +"alexnet","128","1_2_4_5","5951.10" +"resnet50","128","1_2_4_5","1486.53" +"vgg16","128","1_2_4_5","759.77" +"inception3","128","1_2_4_5","971.02" +"alexnet","32","2_3_4_6","1662.92" +"resnet50","32","2_3_4_6","1066.89" +"vgg16","32","2_3_4_6","474.97" +"inception3","32","2_3_4_6","740.54" +"alexnet","64","2_3_4_6","3234.39" +"resnet50","64","2_3_4_6","1302.43" +"vgg16","64","2_3_4_6","631.43" +"inception3","64","2_3_4_6","878.42" +"alexnet","128","2_3_4_6","6004.72" +"resnet50","128","2_3_4_6","1502.46" +"vgg16","128","2_3_4_6","765.93" +"inception3","128","2_3_4_6","970.87" +"alexnet","32","1_2_5_7","2387.33" +"resnet50","32","1_2_5_7","1086.66" +"vgg16","32","1_2_5_7","622.22" +"inception3","32","1_2_5_7","747.42" +"alexnet","64","1_2_5_7","4572.99" +"resnet50","64","1_2_5_7","1309.35" +"vgg16","64","1_2_5_7","762.45" +"inception3","64","1_2_5_7","896.42" +"alexnet","128","1_2_5_7","8047.71" +"resnet50","128","1_2_5_7","1487.27" +"vgg16","128","1_2_5_7","850.19" +"inception3","128","1_2_5_7","954.46" +"alexnet","32","2_3_5_6","1716.47" +"resnet50","32","2_3_5_6","1073.20" +"vgg16","32","2_3_5_6","484.66" +"inception3","32","2_3_5_6","745.13" +"alexnet","64","2_3_5_6","3327.57" +"resnet50","64","2_3_5_6","1286.68" +"vgg16","64","2_3_5_6","649.98" +"inception3","64","2_3_5_6","889.26" +"alexnet","128","2_3_5_6","6203.28" +"resnet50","128","2_3_5_6","1481.07" +"vgg16","128","2_3_5_6","761.74" +"inception3","128","2_3_5_6","973.17" +"alexnet","32","0_3_4_6_7","2096.95" +"resnet50","32","0_3_4_6_7","1244.62" +"vgg16","32","0_3_4_6_7","590.19" +"inception3","32","0_3_4_6_7","890.80" +"alexnet","64","0_3_4_6_7","4038.73" +"resnet50","64","0_3_4_6_7","1587.11" +"vgg16","64","0_3_4_6_7","787.02" +"inception3","64","0_3_4_6_7","1087.92" +"alexnet","128","0_3_4_6_7","7385.17" +"resnet50","128","0_3_4_6_7","1807.63" +"vgg16","128","0_3_4_6_7","901.62" +"inception3","128","0_3_4_6_7","1188.98" +"alexnet","32","1_2_4_5_6","2045.66" +"resnet50","32","1_2_4_5_6","1233.05" +"vgg16","32","1_2_4_5_6","733.63" +"inception3","32","1_2_4_5_6","912.50" +"alexnet","64","1_2_4_5_6","3937.85" +"resnet50","64","1_2_4_5_6","1519.26" +"vgg16","64","1_2_4_5_6","924.42" +"inception3","64","1_2_4_5_6","1102.64" +"alexnet","128","1_2_4_5_6","7130.98" +"resnet50","128","1_2_4_5_6","1749.82" +"vgg16","128","1_2_4_5_6","1046.62" +"inception3","128","1_2_4_5_6","1210.15" +"alexnet","32","0_1_2_5_7","1633.01" +"resnet50","32","0_1_2_5_7","1294.58" +"vgg16","32","0_1_2_5_7","593.93" +"inception3","32","0_1_2_5_7","907.39" +"alexnet","64","0_1_2_5_7","3217.48" +"resnet50","64","0_1_2_5_7","1453.33" +"vgg16","64","0_1_2_5_7","790.96" +"inception3","64","0_1_2_5_7","1083.60" +"alexnet","128","0_1_2_5_7","5926.63" +"resnet50","128","0_1_2_5_7","1820.01" +"vgg16","128","0_1_2_5_7","928.79" +"inception3","128","0_1_2_5_7","1210.40" +"alexnet","32","0_1_2_3_5","2400.05" +"resnet50","32","0_1_2_3_5","1280.20" +"vgg16","32","0_1_2_3_5","711.67" +"inception3","32","0_1_2_3_5","903.58" +"alexnet","64","0_1_2_3_5","4586.97" +"resnet50","64","0_1_2_3_5","1512.00" +"vgg16","64","0_1_2_3_5","905.37" +"inception3","64","0_1_2_3_5","1084.90" +"alexnet","128","0_1_2_3_5","8123.87" +"resnet50","128","0_1_2_3_5","1730.12" +"vgg16","128","0_1_2_3_5","1025.92" +"inception3","128","0_1_2_3_5","1200.12" +"alexnet","32","0_1_3_6_7","1792.58" +"resnet50","32","0_1_3_6_7","1258.17" +"vgg16","32","0_1_3_6_7","486.53" +"inception3","32","0_1_3_6_7","897.38" +"alexnet","64","0_1_3_6_7","3486.49" +"resnet50","64","0_1_3_6_7","1567.40" +"vgg16","64","0_1_3_6_7","690.39" +"inception3","64","0_1_3_6_7","1076.72" +"alexnet","128","0_1_3_6_7","6414.55" +"resnet50","128","0_1_3_6_7","1724.28" +"vgg16","128","0_1_3_6_7","862.16" +"inception3","128","0_1_3_6_7","1182.86" +"alexnet","32","0_1_2_3_6","2747.52" +"resnet50","32","0_1_2_3_6","1303.52" +"vgg16","32","0_1_2_3_6","692.67" +"inception3","32","0_1_2_3_6","904.00" +"alexnet","64","0_1_2_3_6","5232.22" +"resnet50","64","0_1_2_3_6","1466.35" +"vgg16","64","0_1_2_3_6","878.76" +"inception3","64","0_1_2_3_6","1074.37" +"alexnet","128","0_1_2_3_6","9005.99" +"resnet50","128","0_1_2_3_6","1736.32" +"vgg16","128","0_1_2_3_6","972.20" +"inception3","128","0_1_2_3_6","1197.24" +"alexnet","32","0_1_3_4_7","2878.23" +"resnet50","32","0_1_3_4_7","1279.76" +"vgg16","32","0_1_3_4_7","605.28" +"inception3","32","0_1_3_4_7","909.46" +"alexnet","64","0_1_3_4_7","5433.16" +"resnet50","64","0_1_3_4_7","1551.98" +"vgg16","64","0_1_3_4_7","816.51" +"inception3","64","0_1_3_4_7","1082.58" +"alexnet","128","0_1_3_4_7","9513.94" +"resnet50","128","0_1_3_4_7","1808.61" +"vgg16","128","0_1_3_4_7","917.26" +"inception3","128","0_1_3_4_7","1181.58" +"alexnet","32","0_3_4_5_7","2070.60" +"resnet50","32","0_3_4_5_7","1294.73" +"vgg16","32","0_3_4_5_7","569.90" +"inception3","32","0_3_4_5_7","907.48" +"alexnet","64","0_3_4_5_7","4016.00" +"resnet50","64","0_3_4_5_7","1482.22" +"vgg16","64","0_3_4_5_7","778.48" +"inception3","64","0_3_4_5_7","1095.07" +"alexnet","128","0_3_4_5_7","7327.48" +"resnet50","128","0_3_4_5_7","1662.35" +"vgg16","128","0_3_4_5_7","905.78" +"inception3","128","0_3_4_5_7","1197.05" +"alexnet","32","3_4_5_6_7","1551.26" +"resnet50","32","3_4_5_6_7","1302.50" +"vgg16","32","3_4_5_6_7","731.90" +"inception3","32","3_4_5_6_7","908.88" +"alexnet","64","3_4_5_6_7","3082.40" +"resnet50","64","3_4_5_6_7","1574.96" +"vgg16","64","3_4_5_6_7","934.12" +"inception3","64","3_4_5_6_7","1092.49" +"alexnet","128","3_4_5_6_7","5733.47" +"resnet50","128","3_4_5_6_7","1737.14" +"vgg16","128","3_4_5_6_7","1046.69" +"inception3","128","3_4_5_6_7","1187.95" +"alexnet","32","1_3_4_5_7","2006.56" +"resnet50","32","1_3_4_5_7","1256.90" +"vgg16","32","1_3_4_5_7","702.96" +"inception3","32","1_3_4_5_7","912.71" +"alexnet","64","1_3_4_5_7","3842.77" +"resnet50","64","1_3_4_5_7","1607.00" +"vgg16","64","1_3_4_5_7","904.43" +"inception3","64","1_3_4_5_7","1081.03" +"alexnet","128","1_3_4_5_7","6934.98" +"resnet50","128","1_3_4_5_7","1676.79" +"vgg16","128","1_3_4_5_7","1021.92" +"inception3","128","1_3_4_5_7","1203.02" +"alexnet","32","1_3_4_6_7","1501.06" +"resnet50","32","1_3_4_6_7","1270.40" +"vgg16","32","1_3_4_6_7","576.07" +"inception3","32","1_3_4_6_7","891.71" +"alexnet","64","1_3_4_6_7","2944.82" +"resnet50","64","1_3_4_6_7","1462.29" +"vgg16","64","1_3_4_6_7","767.65" +"inception3","64","1_3_4_6_7","1095.94" +"alexnet","128","1_3_4_6_7","5532.55" +"resnet50","128","1_3_4_6_7","1739.56" +"vgg16","128","1_3_4_6_7","903.32" +"inception3","128","1_3_4_6_7","1192.67" +"alexnet","32","0_2_4_5_6","2000.13" +"resnet50","32","0_2_4_5_6","1239.91" +"vgg16","32","0_2_4_5_6","581.25" +"inception3","32","0_2_4_5_6","902.46" +"alexnet","64","0_2_4_5_6","3880.47" +"resnet50","64","0_2_4_5_6","1614.45" +"vgg16","64","0_2_4_5_6","788.24" +"inception3","64","0_2_4_5_6","1102.10" +"alexnet","128","0_2_4_5_6","7162.67" +"resnet50","128","0_2_4_5_6","1717.21" +"vgg16","128","0_2_4_5_6","916.92" +"inception3","128","0_2_4_5_6","1216.70" +"alexnet","32","1_2_3_4_7","1866.07" +"resnet50","32","1_2_3_4_7","1223.27" +"vgg16","32","1_2_3_4_7","501.97" +"inception3","32","1_2_3_4_7","912.99" +"alexnet","64","1_2_3_4_7","3571.67" +"resnet50","64","1_2_3_4_7","1475.68" +"vgg16","64","1_2_3_4_7","695.44" +"inception3","64","1_2_3_4_7","1096.39" +"alexnet","128","1_2_3_4_7","6556.04" +"resnet50","128","1_2_3_4_7","1675.83" +"vgg16","128","1_2_3_4_7","848.36" +"inception3","128","1_2_3_4_7","1220.71" +"alexnet","32","1_3_4_5_6","1969.43" +"resnet50","32","1_3_4_5_6","1291.81" +"vgg16","32","1_3_4_5_6","717.73" +"inception3","32","1_3_4_5_6","918.01" +"alexnet","64","1_3_4_5_6","3822.32" +"resnet50","64","1_3_4_5_6","1549.91" +"vgg16","64","1_3_4_5_6","916.83" +"inception3","64","1_3_4_5_6","1100.01" +"alexnet","128","1_3_4_5_6","6957.16" +"resnet50","128","1_3_4_5_6","1680.00" +"vgg16","128","1_3_4_5_6","1038.11" +"inception3","128","1_3_4_5_6","1212.12" +"alexnet","32","0_1_4_5_6","1942.22" +"resnet50","32","0_1_4_5_6","1245.53" +"vgg16","32","0_1_4_5_6","718.38" +"inception3","32","0_1_4_5_6","911.16" +"alexnet","64","0_1_4_5_6","3781.73" +"resnet50","64","0_1_4_5_6","1599.24" +"vgg16","64","0_1_4_5_6","924.70" +"inception3","64","0_1_4_5_6","1085.81" +"alexnet","128","0_1_4_5_6","6931.85" +"resnet50","128","0_1_4_5_6","1669.00" +"vgg16","128","0_1_4_5_6","1044.24" +"inception3","128","0_1_4_5_6","1195.83" +"alexnet","32","1_2_5_6_7","2101.63" +"resnet50","32","1_2_5_6_7","1259.52" +"vgg16","32","1_2_5_6_7","732.89" +"inception3","32","1_2_5_6_7","898.59" +"alexnet","64","1_2_5_6_7","4070.02" +"resnet50","64","1_2_5_6_7","1554.38" +"vgg16","64","1_2_5_6_7","928.11" +"inception3","64","1_2_5_6_7","1074.85" +"alexnet","128","1_2_5_6_7","7376.29" +"resnet50","128","1_2_5_6_7","1818.20" +"vgg16","128","1_2_5_6_7","1045.92" +"inception3","128","1_2_5_6_7","1201.52" +"alexnet","32","2_3_4_6_7","1947.96" +"resnet50","32","2_3_4_6_7","1275.36" +"vgg16","32","2_3_4_6_7","697.03" +"inception3","32","2_3_4_6_7","896.69" +"alexnet","64","2_3_4_6_7","3782.46" +"resnet50","64","2_3_4_6_7","1519.12" +"vgg16","64","2_3_4_6_7","902.11" +"inception3","64","2_3_4_6_7","1083.28" +"alexnet","128","2_3_4_6_7","6839.63" +"resnet50","128","2_3_4_6_7","1796.80" +"vgg16","128","2_3_4_6_7","1019.91" +"inception3","128","2_3_4_6_7","1206.84" +"alexnet","32","1_2_3_4_5","2260.75" +"resnet50","32","1_2_3_4_5","1228.11" +"vgg16","32","1_2_3_4_5","479.68" +"inception3","32","1_2_3_4_5","898.90" +"alexnet","64","1_2_3_4_5","4367.22" +"resnet50","64","1_2_3_4_5","1574.11" +"vgg16","64","1_2_3_4_5","681.27" +"inception3","64","1_2_3_4_5","1097.51" +"alexnet","128","1_2_3_4_5","7818.37" +"resnet50","128","1_2_3_4_5","1823.85" +"vgg16","128","1_2_3_4_5","835.01" +"inception3","128","1_2_3_4_5","1211.19" +"alexnet","32","1_2_4_6_7","1541.21" +"resnet50","32","1_2_4_6_7","1260.23" +"vgg16","32","1_2_4_6_7","701.59" +"inception3","32","1_2_4_6_7","890.74" +"alexnet","64","1_2_4_6_7","3007.49" +"resnet50","64","1_2_4_6_7","1574.34" +"vgg16","64","1_2_4_6_7","906.03" +"inception3","64","1_2_4_6_7","1079.29" +"alexnet","128","1_2_4_6_7","5700.88" +"resnet50","128","1_2_4_6_7","1668.05" +"vgg16","128","1_2_4_6_7","1024.56" +"inception3","128","1_2_4_6_7","1200.23" +"alexnet","32","2_3_4_5_6","1953.85" +"resnet50","32","2_3_4_5_6","1304.47" +"vgg16","32","2_3_4_5_6","574.17" +"inception3","32","2_3_4_5_6","913.96" +"alexnet","64","2_3_4_5_6","3763.69" +"resnet50","64","2_3_4_5_6","1453.61" +"vgg16","64","2_3_4_5_6","783.19" +"inception3","64","2_3_4_5_6","1084.33" +"alexnet","128","2_3_4_5_6","6870.74" +"resnet50","128","2_3_4_5_6","1812.86" +"vgg16","128","2_3_4_5_6","921.20" +"inception3","128","2_3_4_5_6","1215.62" +"alexnet","32","1_2_4_5_7","2047.07" +"resnet50","32","1_2_4_5_7","1286.74" +"vgg16","32","1_2_4_5_7","699.49" +"inception3","32","1_2_4_5_7","901.24" +"alexnet","64","1_2_4_5_7","3951.68" +"resnet50","64","1_2_4_5_7","1608.10" +"vgg16","64","1_2_4_5_7","903.56" +"inception3","64","1_2_4_5_7","1093.76" +"alexnet","128","1_2_4_5_7","7173.19" +"resnet50","128","1_2_4_5_7","1686.07" +"vgg16","128","1_2_4_5_7","1022.74" +"inception3","128","1_2_4_5_7","1183.73" +"alexnet","32","1_2_3_5_7","2897.10" +"resnet50","32","1_2_3_5_7","1242.68" +"vgg16","32","1_2_3_5_7","596.50" +"inception3","32","1_2_3_5_7","920.73" +"alexnet","64","1_2_3_5_7","5433.04" +"resnet50","64","1_2_3_5_7","1597.46" +"vgg16","64","1_2_3_5_7","797.26" +"inception3","64","1_2_3_5_7","1100.52" +"alexnet","128","1_2_3_5_7","9612.76" +"resnet50","128","1_2_3_5_7","1830.24" +"vgg16","128","1_2_3_5_7","907.01" +"inception3","128","1_2_3_5_7","1220.14" +"alexnet","32","1_4_5_6_7","1603.67" +"resnet50","32","1_4_5_6_7","1291.48" +"vgg16","32","1_4_5_6_7","726.37" +"inception3","32","1_4_5_6_7","911.02" +"alexnet","64","1_4_5_6_7","3113.77" +"resnet50","64","1_4_5_6_7","1585.38" +"vgg16","64","1_4_5_6_7","915.06" +"inception3","64","1_4_5_6_7","1096.72" +"alexnet","128","1_4_5_6_7","5931.87" +"resnet50","128","1_4_5_6_7","1822.17" +"vgg16","128","1_4_5_6_7","1028.02" +"inception3","128","1_4_5_6_7","1193.14" +"alexnet","32","0_1_2_5_6","1813.13" +"resnet50","32","0_1_2_5_6","1296.12" +"vgg16","32","0_1_2_5_6","606.32" +"inception3","32","0_1_2_5_6","910.30" +"alexnet","64","0_1_2_5_6","3494.02" +"resnet50","64","0_1_2_5_6","1599.53" +"vgg16","64","0_1_2_5_6","820.42" +"inception3","64","0_1_2_5_6","1079.77" +"alexnet","128","0_1_2_5_6","6392.69" +"resnet50","128","0_1_2_5_6","1716.36" +"vgg16","128","0_1_2_5_6","934.58" +"inception3","128","0_1_2_5_6","1200.88" +"alexnet","32","0_1_2_3_4","3975.48" +"resnet50","32","0_1_2_3_4","1253.12" +"vgg16","32","0_1_2_3_4","701.85" +"inception3","32","0_1_2_3_4","897.48" +"alexnet","64","0_1_2_3_4","7339.87" +"resnet50","64","0_1_2_3_4","1607.19" +"vgg16","64","0_1_2_3_4","894.14" +"inception3","64","0_1_2_3_4","1063.87" +"alexnet","128","0_1_2_3_4","11499.26" +"resnet50","128","0_1_2_3_4","1772.07" +"vgg16","128","0_1_2_3_4","1020.44" +"inception3","128","0_1_2_3_4","1216.59" +"alexnet","32","0_1_2_3_7","2454.76" +"resnet50","32","0_1_2_3_7","1306.53" +"vgg16","32","0_1_2_3_7","862.78" +"inception3","32","0_1_2_3_7","897.13" +"alexnet","64","0_1_2_3_7","4557.99" +"resnet50","64","0_1_2_3_7","1546.31" +"vgg16","64","0_1_2_3_7","1021.36" +"inception3","64","0_1_2_3_7","1082.21" +"alexnet","128","0_1_2_3_7","8140.78" +"resnet50","128","0_1_2_3_7","1735.50" +"vgg16","128","0_1_2_3_7","1108.50" +"inception3","128","0_1_2_3_7","1195.33" +"alexnet","32","0_2_3_5_6","1716.98" +"resnet50","32","0_2_3_5_6","1258.91" +"vgg16","32","0_2_3_5_6","498.76" +"inception3","32","0_2_3_5_6","922.09" +"alexnet","64","0_2_3_5_6","3301.12" +"resnet50","64","0_2_3_5_6","1507.59" +"vgg16","64","0_2_3_5_6","689.59" +"inception3","64","0_2_3_5_6","1107.12" +"alexnet","128","0_2_3_5_6","6003.82" +"resnet50","128","0_2_3_5_6","1722.08" +"vgg16","128","0_2_3_5_6","820.35" +"inception3","128","0_2_3_5_6","1222.28" +"alexnet","32","0_2_3_5_7","1862.54" +"resnet50","32","0_2_3_5_7","1219.84" +"vgg16","32","0_2_3_5_7","480.12" +"inception3","32","0_2_3_5_7","917.75" +"alexnet","64","0_2_3_5_7","3588.53" +"resnet50","64","0_2_3_5_7","1597.62" +"vgg16","64","0_2_3_5_7","685.61" +"inception3","64","0_2_3_5_7","1101.13" +"alexnet","128","0_2_3_5_7","6535.39" +"resnet50","128","0_2_3_5_7","1824.97" +"vgg16","128","0_2_3_5_7","847.97" +"inception3","128","0_2_3_5_7","1201.51" +"alexnet","32","1_3_5_6_7","2028.19" +"resnet50","32","1_3_5_6_7","1270.26" +"vgg16","32","1_3_5_6_7","598.51" +"inception3","32","1_3_5_6_7","908.74" +"alexnet","64","1_3_5_6_7","3924.72" +"resnet50","64","1_3_5_6_7","1583.99" +"vgg16","64","1_3_5_6_7","808.09" +"inception3","64","1_3_5_6_7","1074.91" +"alexnet","128","1_3_5_6_7","7191.10" +"resnet50","128","1_3_5_6_7","1819.90" +"vgg16","128","1_3_5_6_7","915.02" +"inception3","128","1_3_5_6_7","1174.29" +"alexnet","32","0_1_2_4_7","2225.38" +"resnet50","32","0_1_2_4_7","1239.60" +"vgg16","32","0_1_2_4_7","608.73" +"inception3","32","0_1_2_4_7","903.68" +"alexnet","64","0_1_2_4_7","4286.77" +"resnet50","64","0_1_2_4_7","1502.18" +"vgg16","64","0_1_2_4_7","806.50" +"inception3","64","0_1_2_4_7","1079.52" +"alexnet","128","0_1_2_4_7","7752.77" +"resnet50","128","0_1_2_4_7","1820.19" +"vgg16","128","0_1_2_4_7","935.89" +"inception3","128","0_1_2_4_7","1190.31" +"alexnet","32","0_2_3_4_6","2308.84" +"resnet50","32","0_2_3_4_6","1291.10" +"vgg16","32","0_2_3_4_6","575.20" +"inception3","32","0_2_3_4_6","918.15" +"alexnet","64","0_2_3_4_6","4453.95" +"resnet50","64","0_2_3_4_6","1589.10" +"vgg16","64","0_2_3_4_6","764.42" +"inception3","64","0_2_3_4_6","1099.41" +"alexnet","128","0_2_3_4_6","7738.27" +"resnet50","128","0_2_3_4_6","1850.12" +"vgg16","128","0_2_3_4_6","911.52" +"inception3","128","0_2_3_4_6","1220.61" +"alexnet","32","1_2_3_6_7","1795.92" +"resnet50","32","1_2_3_6_7","1264.08" +"vgg16","32","1_2_3_6_7","588.18" +"inception3","32","1_2_3_6_7","885.45" +"alexnet","64","1_2_3_6_7","3495.89" +"resnet50","64","1_2_3_6_7","1566.70" +"vgg16","64","1_2_3_6_7","793.96" +"inception3","64","1_2_3_6_7","1086.36" +"alexnet","128","1_2_3_6_7","6328.07" +"resnet50","128","1_2_3_6_7","1810.12" +"vgg16","128","1_2_3_6_7","908.03" +"inception3","128","1_2_3_6_7","1204.87" +"alexnet","32","2_4_5_6_7","1532.96" +"resnet50","32","2_4_5_6_7","1308.29" +"vgg16","32","2_4_5_6_7","848.21" +"inception3","32","2_4_5_6_7","888.48" +"alexnet","64","2_4_5_6_7","2995.40" +"resnet50","64","2_4_5_6_7","1521.41" +"vgg16","64","2_4_5_6_7","1010.29" +"inception3","64","2_4_5_6_7","1065.38" +"alexnet","128","2_4_5_6_7","5640.42" +"resnet50","128","2_4_5_6_7","1819.17" +"vgg16","128","2_4_5_6_7","1103.84" +"inception3","128","2_4_5_6_7","1188.76" +"alexnet","32","0_3_4_5_6","2037.81" +"resnet50","32","0_3_4_5_6","1238.72" +"vgg16","32","0_3_4_5_6","575.54" +"inception3","32","0_3_4_5_6","921.91" +"alexnet","64","0_3_4_5_6","3947.48" +"resnet50","64","0_3_4_5_6","1581.33" +"vgg16","64","0_3_4_5_6","785.54" +"inception3","64","0_3_4_5_6","1113.43" +"alexnet","128","0_3_4_5_6","7286.87" +"resnet50","128","0_3_4_5_6","1837.29" +"vgg16","128","0_3_4_5_6","927.01" +"inception3","128","0_3_4_5_6","1204.41" +"alexnet","32","2_3_5_6_7","1929.71" +"resnet50","32","2_3_5_6_7","1288.97" +"vgg16","32","2_3_5_6_7","720.06" +"inception3","32","2_3_5_6_7","903.91" +"alexnet","64","2_3_5_6_7","3745.20" +"resnet50","64","2_3_5_6_7","1592.49" +"vgg16","64","2_3_5_6_7","918.45" +"inception3","64","2_3_5_6_7","1088.83" +"alexnet","128","2_3_5_6_7","6819.02" +"resnet50","128","2_3_5_6_7","1811.84" +"vgg16","128","2_3_5_6_7","1036.38" +"inception3","128","2_3_5_6_7","1210.50" +"alexnet","32","0_2_3_6_7","1801.52" +"resnet50","32","0_2_3_6_7","1258.58" +"vgg16","32","0_2_3_6_7","586.08" +"inception3","32","0_2_3_6_7","912.11" +"alexnet","64","0_2_3_6_7","3494.92" +"resnet50","64","0_2_3_6_7","1570.03" +"vgg16","64","0_2_3_6_7","782.41" +"inception3","64","0_2_3_6_7","1086.17" +"alexnet","128","0_2_3_6_7","6350.27" +"resnet50","128","0_2_3_6_7","1813.90" +"vgg16","128","0_2_3_6_7","904.44" +"inception3","128","0_2_3_6_7","1203.68" +"alexnet","32","0_2_5_6_7","1508.65" +"resnet50","32","0_2_5_6_7","1262.68" +"vgg16","32","0_2_5_6_7","722.83" +"inception3","32","0_2_5_6_7","901.51" +"alexnet","64","0_2_5_6_7","2953.61" +"resnet50","64","0_2_5_6_7","1568.43" +"vgg16","64","0_2_5_6_7","921.60" +"inception3","64","0_2_5_6_7","1084.05" +"alexnet","128","0_2_5_6_7","5540.83" +"resnet50","128","0_2_5_6_7","1661.66" +"vgg16","128","0_2_5_6_7","1034.61" +"inception3","128","0_2_5_6_7","1182.24" +"alexnet","32","0_1_3_4_6","2308.20" +"resnet50","32","0_1_3_4_6","1269.52" +"vgg16","32","0_1_3_4_6","580.66" +"inception3","32","0_1_3_4_6","904.28" +"alexnet","64","0_1_3_4_6","4373.01" +"resnet50","64","0_1_3_4_6","1589.51" +"vgg16","64","0_1_3_4_6","786.99" +"inception3","64","0_1_3_4_6","1083.95" +"alexnet","128","0_1_3_4_6","7965.81" +"resnet50","128","0_1_3_4_6","1664.74" +"vgg16","128","0_1_3_4_6","941.74" +"inception3","128","0_1_3_4_6","1192.98" +"alexnet","32","2_3_4_5_7","1523.02" +"resnet50","32","2_3_4_5_7","1232.37" +"vgg16","32","2_3_4_5_7","562.17" +"inception3","32","2_3_4_5_7","893.30" +"alexnet","64","2_3_4_5_7","2982.99" +"resnet50","64","2_3_4_5_7","1600.11" +"vgg16","64","2_3_4_5_7","747.26" +"inception3","64","2_3_4_5_7","1091.18" +"alexnet","128","2_3_4_5_7","5516.58" +"resnet50","128","2_3_4_5_7","1718.25" +"vgg16","128","2_3_4_5_7","899.51" +"inception3","128","2_3_4_5_7","1207.92" +"alexnet","32","0_2_4_5_7","1969.53" +"resnet50","32","0_2_4_5_7","1290.92" +"vgg16","32","0_2_4_5_7","565.11" +"inception3","32","0_2_4_5_7","903.96" +"alexnet","64","0_2_4_5_7","3864.06" +"resnet50","64","0_2_4_5_7","1541.56" +"vgg16","64","0_2_4_5_7","763.79" +"inception3","64","0_2_4_5_7","1094.76" +"alexnet","128","0_2_4_5_7","7134.71" +"resnet50","128","0_2_4_5_7","1811.45" +"vgg16","128","0_2_4_5_7","922.02" +"inception3","128","0_2_4_5_7","1204.41" +"alexnet","32","0_1_2_4_5","2092.77" +"resnet50","32","0_1_2_4_5","1258.84" +"vgg16","32","0_1_2_4_5","587.64" +"inception3","32","0_1_2_4_5","893.73" +"alexnet","64","0_1_2_4_5","4131.13" +"resnet50","64","0_1_2_4_5","1471.17" +"vgg16","64","0_1_2_4_5","762.08" +"inception3","64","0_1_2_4_5","1072.60" +"alexnet","128","0_1_2_4_5","7380.87" +"resnet50","128","0_1_2_4_5","1794.98" +"vgg16","128","0_1_2_4_5","883.78" +"inception3","128","0_1_2_4_5","1205.11" +"alexnet","32","0_1_2_4_6","2738.88" +"resnet50","32","0_1_2_4_6","1246.36" +"vgg16","32","0_1_2_4_6","583.17" +"inception3","32","0_1_2_4_6","918.12" +"alexnet","64","0_1_2_4_6","5234.36" +"resnet50","64","0_1_2_4_6","1474.60" +"vgg16","64","0_1_2_4_6","798.93" +"inception3","64","0_1_2_4_6","1087.87" +"alexnet","128","0_1_2_4_6","9053.17" +"resnet50","128","0_1_2_4_6","1826.32" +"vgg16","128","0_1_2_4_6","940.27" +"inception3","128","0_1_2_4_6","1200.63" +"alexnet","32","0_2_3_4_5","2166.06" +"resnet50","32","0_2_3_4_5","1227.28" +"vgg16","32","0_2_3_4_5","587.18" +"inception3","32","0_2_3_4_5","898.74" +"alexnet","64","0_2_3_4_5","4135.93" +"resnet50","64","0_2_3_4_5","1563.88" +"vgg16","64","0_2_3_4_5","794.15" +"inception3","64","0_2_3_4_5","1092.84" +"alexnet","128","0_2_3_4_5","7566.69" +"resnet50","128","0_2_3_4_5","1684.63" +"vgg16","128","0_2_3_4_5","940.31" +"inception3","128","0_2_3_4_5","1210.55" +"alexnet","32","0_1_5_6_7","1495.78" +"resnet50","32","0_1_5_6_7","1244.62" +"vgg16","32","0_1_5_6_7","589.59" +"inception3","32","0_1_5_6_7","895.15" +"alexnet","64","0_1_5_6_7","2946.50" +"resnet50","64","0_1_5_6_7","1497.50" +"vgg16","64","0_1_5_6_7","807.82" +"inception3","64","0_1_5_6_7","1075.77" +"alexnet","128","0_1_5_6_7","5440.57" +"resnet50","128","0_1_5_6_7","1802.65" +"vgg16","128","0_1_5_6_7","907.90" +"inception3","128","0_1_5_6_7","1192.22" +"alexnet","32","0_4_5_6_7","1585.16" +"resnet50","32","0_4_5_6_7","1272.69" +"vgg16","32","0_4_5_6_7","738.65" +"inception3","32","0_4_5_6_7","905.05" +"alexnet","64","0_4_5_6_7","3125.42" +"resnet50","64","0_4_5_6_7","1539.23" +"vgg16","64","0_4_5_6_7","932.79" +"inception3","64","0_4_5_6_7","1078.71" +"alexnet","128","0_4_5_6_7","5823.83" +"resnet50","128","0_4_5_6_7","1824.26" +"vgg16","128","0_4_5_6_7","1054.81" +"inception3","128","0_4_5_6_7","1183.67" +"alexnet","32","0_1_3_4_5","2187.28" +"resnet50","32","0_1_3_4_5","1252.56" +"vgg16","32","0_1_3_4_5","588.21" +"inception3","32","0_1_3_4_5","903.21" +"alexnet","64","0_1_3_4_5","4174.69" +"resnet50","64","0_1_3_4_5","1554.63" +"vgg16","64","0_1_3_4_5","781.55" +"inception3","64","0_1_3_4_5","1069.52" +"alexnet","128","0_1_3_4_5","7531.90" +"resnet50","128","0_1_3_4_5","1828.37" +"vgg16","128","0_1_3_4_5","864.97" +"inception3","128","0_1_3_4_5","1197.79" +"alexnet","32","0_1_4_6_7","1952.57" +"resnet50","32","0_1_4_6_7","1240.84" +"vgg16","32","0_1_4_6_7","576.15" +"inception3","32","0_1_4_6_7","891.27" +"alexnet","64","0_1_4_6_7","3769.86" +"resnet50","64","0_1_4_6_7","1569.08" +"vgg16","64","0_1_4_6_7","780.82" +"inception3","64","0_1_4_6_7","1076.43" +"alexnet","128","0_1_4_6_7","6910.49" +"resnet50","128","0_1_4_6_7","1812.87" +"vgg16","128","0_1_4_6_7","911.72" +"inception3","128","0_1_4_6_7","1186.93" +"alexnet","32","0_1_4_5_7","1958.31" +"resnet50","32","0_1_4_5_7","1248.38" +"vgg16","32","0_1_4_5_7","699.27" +"inception3","32","0_1_4_5_7","905.09" +"alexnet","64","0_1_4_5_7","3787.95" +"resnet50","64","0_1_4_5_7","1462.07" +"vgg16","64","0_1_4_5_7","903.41" +"inception3","64","0_1_4_5_7","1073.80" +"alexnet","128","0_1_4_5_7","6910.22" +"resnet50","128","0_1_4_5_7","1833.03" +"vgg16","128","0_1_4_5_7","1027.51" +"inception3","128","0_1_4_5_7","1197.57" +"alexnet","32","0_1_3_5_7","1864.89" +"resnet50","32","0_1_3_5_7","1300.54" +"vgg16","32","0_1_3_5_7","591.89" +"inception3","32","0_1_3_5_7","897.85" +"alexnet","64","0_1_3_5_7","3596.49" +"resnet50","64","0_1_3_5_7","1459.54" +"vgg16","64","0_1_3_5_7","778.05" +"inception3","64","0_1_3_5_7","1087.32" +"alexnet","128","0_1_3_5_7","6518.84" +"resnet50","128","0_1_3_5_7","1644.44" +"vgg16","128","0_1_3_5_7","919.46" +"inception3","128","0_1_3_5_7","1164.29" +"alexnet","32","0_2_4_6_7","2005.30" +"resnet50","32","0_2_4_6_7","1257.13" +"vgg16","32","0_2_4_6_7","708.18" +"inception3","32","0_2_4_6_7","899.03" +"alexnet","64","0_2_4_6_7","3887.20" +"resnet50","64","0_2_4_6_7","1516.00" +"vgg16","64","0_2_4_6_7","911.42" +"inception3","64","0_2_4_6_7","1074.13" +"alexnet","128","0_2_4_6_7","7207.91" +"resnet50","128","0_2_4_6_7","1668.95" +"vgg16","128","0_2_4_6_7","1034.98" +"inception3","128","0_2_4_6_7","1197.73" +"alexnet","32","1_2_3_4_6","1697.41" +"resnet50","32","1_2_3_4_6","1295.04" +"vgg16","32","1_2_3_4_6","478.41" +"inception3","32","1_2_3_4_6","896.31" +"alexnet","64","1_2_3_4_6","3296.38" +"resnet50","64","1_2_3_4_6","1482.89" +"vgg16","64","1_2_3_4_6","680.43" +"inception3","64","1_2_3_4_6","1102.41" +"alexnet","128","1_2_3_4_6","6001.96" +"resnet50","128","1_2_3_4_6","1733.63" +"vgg16","128","1_2_3_4_6","813.46" +"inception3","128","1_2_3_4_6","1212.84" +"alexnet","32","0_1_3_5_6","1675.89" +"resnet50","32","0_1_3_5_6","1276.26" +"vgg16","32","0_1_3_5_6","611.55" +"inception3","32","0_1_3_5_6","901.43" +"alexnet","64","0_1_3_5_6","3264.97" +"resnet50","64","0_1_3_5_6","1589.85" +"vgg16","64","0_1_3_5_6","812.13" +"inception3","64","0_1_3_5_6","1083.70" +"alexnet","128","0_1_3_5_6","6061.91" +"resnet50","128","0_1_3_5_6","1838.97" +"vgg16","128","0_1_3_5_6","951.21" +"inception3","128","0_1_3_5_6","1189.52" +"alexnet","32","0_1_2_6_7","1706.97" +"resnet50","32","0_1_2_6_7","1258.15" +"vgg16","32","0_1_2_6_7","590.29" +"inception3","32","0_1_2_6_7","897.56" +"alexnet","64","0_1_2_6_7","3346.29" +"resnet50","64","0_1_2_6_7","1563.33" +"vgg16","64","0_1_2_6_7","802.02" +"inception3","64","0_1_2_6_7","1088.17" +"alexnet","128","0_1_2_6_7","6156.87" +"resnet50","128","0_1_2_6_7","1798.12" +"vgg16","128","0_1_2_6_7","930.58" +"inception3","128","0_1_2_6_7","1190.65" +"alexnet","32","0_3_5_6_7","1552.67" +"resnet50","32","0_3_5_6_7","1284.28" +"vgg16","32","0_3_5_6_7","596.95" +"inception3","32","0_3_5_6_7","889.79" +"alexnet","64","0_3_5_6_7","3032.89" +"resnet50","64","0_3_5_6_7","1571.15" +"vgg16","64","0_3_5_6_7","781.86" +"inception3","64","0_3_5_6_7","1085.92" +"alexnet","128","0_3_5_6_7","5719.18" +"resnet50","128","0_3_5_6_7","1805.82" +"vgg16","128","0_3_5_6_7","922.14" +"inception3","128","0_3_5_6_7","1191.38" +"alexnet","32","0_2_3_4_7","2894.01" +"resnet50","32","0_2_3_4_7","1260.65" +"vgg16","32","0_2_3_4_7","605.23" +"inception3","32","0_2_3_4_7","925.30" +"alexnet","64","0_2_3_4_7","5506.22" +"resnet50","64","0_2_3_4_7","1594.01" +"vgg16","64","0_2_3_4_7","796.55" +"inception3","64","0_2_3_4_7","1100.56" +"alexnet","128","0_2_3_4_7","9759.62" +"resnet50","128","0_2_3_4_7","1752.98" +"vgg16","128","0_2_3_4_7","948.43" +"inception3","128","0_2_3_4_7","1191.96" +"alexnet","32","1_2_3_5_6","2303.02" +"resnet50","32","1_2_3_5_6","1307.87" +"vgg16","32","1_2_3_5_6","595.21" +"inception3","32","1_2_3_5_6","916.48" +"alexnet","64","1_2_3_5_6","4401.84" +"resnet50","64","1_2_3_5_6","1610.75" +"vgg16","64","1_2_3_5_6","787.11" +"inception3","64","1_2_3_5_6","1095.32" +"alexnet","128","1_2_3_5_6","7728.42" +"resnet50","128","1_2_3_5_6","1750.55" +"vgg16","128","1_2_3_5_6","927.22" +"inception3","128","1_2_3_5_6","1219.54" +"alexnet","32","0_1_2_3_5_6","2136.18" +"resnet50","32","0_1_2_3_5_6","1477.12" +"vgg16","32","0_1_2_3_5_6","795.63" +"inception3","32","0_1_2_3_5_6","1026.29" +"alexnet","64","0_1_2_3_5_6","4203.41" +"resnet50","64","0_1_2_3_5_6","1872.17" +"vgg16","64","0_1_2_3_5_6","1040.37" +"inception3","64","0_1_2_3_5_6","1269.95" +"alexnet","128","0_1_2_3_5_6","7960.30" +"resnet50","128","0_1_2_3_5_6","2077.50" +"vgg16","128","0_1_2_3_5_6","1185.85" +"inception3","128","0_1_2_3_5_6","1385.35" +"alexnet","32","0_1_2_3_6_7","2546.82" +"resnet50","32","0_1_2_3_6_7","1479.26" +"vgg16","32","0_1_2_3_6_7","765.25" +"inception3","32","0_1_2_3_6_7","1013.84" +"alexnet","64","0_1_2_3_6_7","5013.28" +"resnet50","64","0_1_2_3_6_7","1853.81" +"vgg16","64","0_1_2_3_6_7","1008.50" +"inception3","64","0_1_2_3_6_7","1238.89" +"alexnet","128","0_1_2_3_6_7","9306.88" +"resnet50","128","0_1_2_3_6_7","2117.93" +"vgg16","128","0_1_2_3_6_7","1162.93" +"inception3","128","0_1_2_3_6_7","1328.98" +"alexnet","32","0_1_3_4_5_6","2241.22" +"resnet50","32","0_1_3_4_5_6","1467.36" +"vgg16","32","0_1_3_4_5_6","535.34" +"inception3","32","0_1_3_4_5_6","1060.03" +"alexnet","64","0_1_3_4_5_6","4425.51" +"resnet50","64","0_1_3_4_5_6","1761.41" +"vgg16","64","0_1_3_4_5_6","743.15" +"inception3","64","0_1_3_4_5_6","1248.30" +"alexnet","128","0_1_3_4_5_6","8614.10" +"resnet50","128","0_1_3_4_5_6","2048.26" +"vgg16","128","0_1_3_4_5_6","911.30" +"inception3","128","0_1_3_4_5_6","1410.49" +"alexnet","32","0_1_3_5_6_7","2418.94" +"resnet50","32","0_1_3_5_6_7","1430.26" +"vgg16","32","0_1_3_5_6_7","660.24" +"inception3","32","0_1_3_5_6_7","1001.87" +"alexnet","64","0_1_3_5_6_7","4781.64" +"resnet50","64","0_1_3_5_6_7","1839.43" +"vgg16","64","0_1_3_5_6_7","889.03" +"inception3","64","0_1_3_5_6_7","1242.76" +"alexnet","128","0_1_3_5_6_7","9201.09" +"resnet50","128","0_1_3_5_6_7","2111.45" +"vgg16","128","0_1_3_5_6_7","1007.66" +"inception3","128","0_1_3_5_6_7","1384.83" +"alexnet","32","0_2_3_4_6_7","2599.60" +"resnet50","32","0_2_3_4_6_7","1523.90" +"vgg16","32","0_2_3_4_6_7","666.28" +"inception3","32","0_2_3_4_6_7","1035.90" +"alexnet","64","0_2_3_4_6_7","5097.29" +"resnet50","64","0_2_3_4_6_7","1843.94" +"vgg16","64","0_2_3_4_6_7","890.13" +"inception3","64","0_2_3_4_6_7","1251.51" +"alexnet","128","0_2_3_4_6_7","9231.44" +"resnet50","128","0_2_3_4_6_7","2156.50" +"vgg16","128","0_2_3_4_6_7","1029.38" +"inception3","128","0_2_3_4_6_7","1395.31" +"alexnet","32","0_3_4_5_6_7","2356.49" +"resnet50","32","0_3_4_5_6_7","1488.69" +"vgg16","32","0_3_4_5_6_7","799.16" +"inception3","32","0_3_4_5_6_7","983.01" +"alexnet","64","0_3_4_5_6_7","4698.59" +"resnet50","64","0_3_4_5_6_7","1779.03" +"vgg16","64","0_3_4_5_6_7","1037.36" +"inception3","64","0_3_4_5_6_7","1233.49" +"alexnet","128","0_3_4_5_6_7","8762.42" +"resnet50","128","0_3_4_5_6_7","2064.23" +"vgg16","128","0_3_4_5_6_7","1197.60" +"inception3","128","0_3_4_5_6_7","1400.64" +"alexnet","32","2_3_4_5_6_7","2149.95" +"resnet50","32","2_3_4_5_6_7","1416.79" +"vgg16","32","2_3_4_5_6_7","646.01" +"inception3","32","2_3_4_5_6_7","1040.70" +"alexnet","64","2_3_4_5_6_7","4294.91" +"resnet50","64","2_3_4_5_6_7","1858.07" +"vgg16","64","2_3_4_5_6_7","878.64" +"inception3","64","2_3_4_5_6_7","1245.40" +"alexnet","128","2_3_4_5_6_7","8208.88" +"resnet50","128","2_3_4_5_6_7","2093.12" +"vgg16","128","2_3_4_5_6_7","994.44" +"inception3","128","2_3_4_5_6_7","1372.45" +"alexnet","32","0_1_2_4_6_7","2639.36" +"resnet50","32","0_1_2_4_6_7","1466.42" +"vgg16","32","0_1_2_4_6_7","629.41" +"inception3","32","0_1_2_4_6_7","1033.58" +"alexnet","64","0_1_2_4_6_7","5172.85" +"resnet50","64","0_1_2_4_6_7","1855.82" +"vgg16","64","0_1_2_4_6_7","865.84" +"inception3","64","0_1_2_4_6_7","1231.73" +"alexnet","128","0_1_2_4_6_7","9318.40" +"resnet50","128","0_1_2_4_6_7","2057.89" +"vgg16","128","0_1_2_4_6_7","1009.85" +"inception3","128","0_1_2_4_6_7","1373.66" +"alexnet","32","0_2_4_5_6_7","2318.75" +"resnet50","32","0_2_4_5_6_7","1505.99" +"vgg16","32","0_2_4_5_6_7","809.82" +"inception3","32","0_2_4_5_6_7","1010.54" +"alexnet","64","0_2_4_5_6_7","4624.44" +"resnet50","64","0_2_4_5_6_7","1868.57" +"vgg16","64","0_2_4_5_6_7","1046.38" +"inception3","64","0_2_4_5_6_7","1205.64" +"alexnet","128","0_2_4_5_6_7","8750.44" +"resnet50","128","0_2_4_5_6_7","2118.04" +"vgg16","128","0_2_4_5_6_7","1212.84" +"inception3","128","0_2_4_5_6_7","1372.23" +"alexnet","32","0_2_3_5_6_7","2020.79" +"resnet50","32","0_2_3_5_6_7","1485.21" +"vgg16","32","0_2_3_5_6_7","669.46" +"inception3","32","0_2_3_5_6_7","1029.40" +"alexnet","64","0_2_3_5_6_7","4011.09" +"resnet50","64","0_2_3_5_6_7","1866.78" +"vgg16","64","0_2_3_5_6_7","883.43" +"inception3","64","0_2_3_5_6_7","1232.02" +"alexnet","128","0_2_3_5_6_7","7835.87" +"resnet50","128","0_2_3_5_6_7","2060.27" +"vgg16","128","0_2_3_5_6_7","1011.93" +"inception3","128","0_2_3_5_6_7","1365.37" +"alexnet","32","1_2_3_4_5_7","2174.13" +"resnet50","32","1_2_3_4_5_7","1487.84" +"vgg16","32","1_2_3_4_5_7","647.28" +"inception3","32","1_2_3_4_5_7","1051.57" +"alexnet","64","1_2_3_4_5_7","4298.73" +"resnet50","64","1_2_3_4_5_7","1833.19" +"vgg16","64","1_2_3_4_5_7","868.72" +"inception3","64","1_2_3_4_5_7","1261.89" +"alexnet","128","1_2_3_4_5_7","8003.76" +"resnet50","128","1_2_3_4_5_7","2047.40" +"vgg16","128","1_2_3_4_5_7","1020.25" +"inception3","128","1_2_3_4_5_7","1395.33" +"alexnet","32","0_1_4_5_6_7","2771.85" +"resnet50","32","0_1_4_5_6_7","1470.31" +"vgg16","32","0_1_4_5_6_7","786.06" +"inception3","32","0_1_4_5_6_7","1010.25" +"alexnet","64","0_1_4_5_6_7","5511.62" +"resnet50","64","0_1_4_5_6_7","1756.03" +"vgg16","64","0_1_4_5_6_7","1028.21" +"inception3","64","0_1_4_5_6_7","1219.35" +"alexnet","128","0_1_4_5_6_7","10000.94" +"resnet50","128","0_1_4_5_6_7","2056.13" +"vgg16","128","0_1_4_5_6_7","1183.95" +"inception3","128","0_1_4_5_6_7","1381.18" +"alexnet","32","0_1_3_4_6_7","2518.95" +"resnet50","32","0_1_3_4_6_7","1469.53" +"vgg16","32","0_1_3_4_6_7","655.05" +"inception3","32","0_1_3_4_6_7","1006.16" +"alexnet","64","0_1_3_4_6_7","5024.04" +"resnet50","64","0_1_3_4_6_7","1855.15" +"vgg16","64","0_1_3_4_6_7","889.98" +"inception3","64","0_1_3_4_6_7","1213.38" +"alexnet","128","0_1_3_4_6_7","9164.86" +"resnet50","128","0_1_3_4_6_7","2130.45" +"vgg16","128","0_1_3_4_6_7","1026.17" +"inception3","128","0_1_3_4_6_7","1390.49" +"alexnet","32","0_1_2_3_4_5","2343.61" +"resnet50","32","0_1_2_3_4_5","1450.73" +"vgg16","32","0_1_2_3_4_5","642.55" +"inception3","32","0_1_2_3_4_5","999.52" +"alexnet","64","0_1_2_3_4_5","4589.08" +"resnet50","64","0_1_2_3_4_5","1828.70" +"vgg16","64","0_1_2_3_4_5","876.93" +"inception3","64","0_1_2_3_4_5","1229.16" +"alexnet","128","0_1_2_3_4_5","8528.08" +"resnet50","128","0_1_2_3_4_5","2152.71" +"vgg16","128","0_1_2_3_4_5","1038.26" +"inception3","128","0_1_2_3_4_5","1396.48" +"alexnet","32","0_1_2_5_6_7","2224.70" +"resnet50","32","0_1_2_5_6_7","1431.83" +"vgg16","32","0_1_2_5_6_7","638.49" +"inception3","32","0_1_2_5_6_7","1028.90" +"alexnet","64","0_1_2_5_6_7","4430.08" +"resnet50","64","0_1_2_5_6_7","1776.42" +"vgg16","64","0_1_2_5_6_7","859.44" +"inception3","64","0_1_2_5_6_7","1235.06" +"alexnet","128","0_1_2_5_6_7","8630.25" +"resnet50","128","0_1_2_5_6_7","2151.30" +"vgg16","128","0_1_2_5_6_7","1004.22" +"inception3","128","0_1_2_5_6_7","1381.84" +"alexnet","32","0_1_2_4_5_6","2346.21" +"resnet50","32","0_1_2_4_5_6","1508.11" +"vgg16","32","0_1_2_4_5_6","659.07" +"inception3","32","0_1_2_4_5_6","1036.47" +"alexnet","64","0_1_2_4_5_6","4648.60" +"resnet50","64","0_1_2_4_5_6","1827.27" +"vgg16","64","0_1_2_4_5_6","872.66" +"inception3","64","0_1_2_4_5_6","1273.45" +"alexnet","128","0_1_2_4_5_6","8477.20" +"resnet50","128","0_1_2_4_5_6","2056.28" +"vgg16","128","0_1_2_4_5_6","1029.73" +"inception3","128","0_1_2_4_5_6","1410.36" +"alexnet","32","1_2_3_4_5_6","1930.17" +"resnet50","32","1_2_3_4_5_6","1451.96" +"vgg16","32","1_2_3_4_5_6","526.10" +"inception3","32","1_2_3_4_5_6","1031.29" +"alexnet","64","1_2_3_4_5_6","3786.73" +"resnet50","64","1_2_3_4_5_6","1887.03" +"vgg16","64","1_2_3_4_5_6","763.42" +"inception3","64","1_2_3_4_5_6","1223.04" +"alexnet","128","1_2_3_4_5_6","7377.01" +"resnet50","128","1_2_3_4_5_6","2063.40" +"vgg16","128","1_2_3_4_5_6","902.95" +"inception3","128","1_2_3_4_5_6","1416.34" +"alexnet","32","1_2_3_5_6_7","2675.43" +"resnet50","32","1_2_3_5_6_7","1499.14" +"vgg16","32","1_2_3_5_6_7","650.54" +"inception3","32","1_2_3_5_6_7","1047.03" +"alexnet","64","1_2_3_5_6_7","5282.57" +"resnet50","64","1_2_3_5_6_7","1881.61" +"vgg16","64","1_2_3_5_6_7","893.08" +"inception3","64","1_2_3_5_6_7","1222.94" +"alexnet","128","1_2_3_5_6_7","9827.73" +"resnet50","128","1_2_3_5_6_7","2150.98" +"vgg16","128","1_2_3_5_6_7","1029.29" +"inception3","128","1_2_3_5_6_7","1381.80" +"alexnet","32","0_2_3_4_5_6","2274.56" +"resnet50","32","0_2_3_4_5_6","1445.00" +"vgg16","32","0_2_3_4_5_6","535.85" +"inception3","32","0_2_3_4_5_6","1041.61" +"alexnet","64","0_2_3_4_5_6","4520.16" +"resnet50","64","0_2_3_4_5_6","1860.40" +"vgg16","64","0_2_3_4_5_6","770.39" +"inception3","64","0_2_3_4_5_6","1253.11" +"alexnet","128","0_2_3_4_5_6","8597.88" +"resnet50","128","0_2_3_4_5_6","2030.67" +"vgg16","128","0_2_3_4_5_6","899.52" +"inception3","128","0_2_3_4_5_6","1376.77" +"alexnet","32","0_1_2_3_4_6","2425.13" +"resnet50","32","0_1_2_3_4_6","1469.15" +"vgg16","32","0_1_2_3_4_6","805.24" +"inception3","32","0_1_2_3_4_6","1032.27" +"alexnet","64","0_1_2_3_4_6","4821.86" +"resnet50","64","0_1_2_3_4_6","1749.76" +"vgg16","64","0_1_2_3_4_6","1042.25" +"inception3","64","0_1_2_3_4_6","1268.70" +"alexnet","128","0_1_2_3_4_6","8864.09" +"resnet50","128","0_1_2_3_4_6","2169.76" +"vgg16","128","0_1_2_3_4_6","1198.34" +"inception3","128","0_1_2_3_4_6","1363.96" +"alexnet","32","0_1_2_4_5_7","2395.86" +"resnet50","32","0_1_2_4_5_7","1423.99" +"vgg16","32","0_1_2_4_5_7","531.80" +"inception3","32","0_1_2_4_5_7","1042.55" +"alexnet","64","0_1_2_4_5_7","4771.69" +"resnet50","64","0_1_2_4_5_7","1786.19" +"vgg16","64","0_1_2_4_5_7","752.23" +"inception3","64","0_1_2_4_5_7","1244.40" +"alexnet","128","0_1_2_4_5_7","8927.83" +"resnet50","128","0_1_2_4_5_7","2130.18" +"vgg16","128","0_1_2_4_5_7","942.76" +"inception3","128","0_1_2_4_5_7","1389.30" +"alexnet","32","0_1_2_3_5_7","2485.30" +"resnet50","32","0_1_2_3_5_7","1512.62" +"vgg16","32","0_1_2_3_5_7","658.25" +"inception3","32","0_1_2_3_5_7","1043.46" +"alexnet","64","0_1_2_3_5_7","4861.86" +"resnet50","64","0_1_2_3_5_7","1855.77" +"vgg16","64","0_1_2_3_5_7","884.39" +"inception3","64","0_1_2_3_5_7","1241.13" +"alexnet","128","0_1_2_3_5_7","9146.08" +"resnet50","128","0_1_2_3_5_7","2173.70" +"vgg16","128","0_1_2_3_5_7","1000.91" +"inception3","128","0_1_2_3_5_7","1393.21" +"alexnet","32","0_2_3_4_5_7","2584.17" +"resnet50","32","0_2_3_4_5_7","1496.48" +"vgg16","32","0_2_3_4_5_7","661.18" +"inception3","32","0_2_3_4_5_7","1037.55" +"alexnet","64","0_2_3_4_5_7","5129.75" +"resnet50","64","0_2_3_4_5_7","1883.96" +"vgg16","64","0_2_3_4_5_7","896.51" +"inception3","64","0_2_3_4_5_7","1220.13" +"alexnet","128","0_2_3_4_5_7","9073.86" +"resnet50","128","0_2_3_4_5_7","2142.97" +"vgg16","128","0_2_3_4_5_7","1041.65" +"inception3","128","0_2_3_4_5_7","1412.48" +"alexnet","32","0_1_3_4_5_7","2482.18" +"resnet50","32","0_1_3_4_5_7","1507.03" +"vgg16","32","0_1_3_4_5_7","658.50" +"inception3","32","0_1_3_4_5_7","1052.52" +"alexnet","64","0_1_3_4_5_7","5025.23" +"resnet50","64","0_1_3_4_5_7","1859.62" +"vgg16","64","0_1_3_4_5_7","888.07" +"inception3","64","0_1_3_4_5_7","1263.87" +"alexnet","128","0_1_3_4_5_7","8985.83" +"resnet50","128","0_1_3_4_5_7","2149.36" +"vgg16","128","0_1_3_4_5_7","982.79" +"inception3","128","0_1_3_4_5_7","1394.34" +"alexnet","32","0_1_2_3_4_7","2985.66" +"resnet50","32","0_1_2_3_4_7","1408.04" +"vgg16","32","0_1_2_3_4_7","660.03" +"inception3","32","0_1_2_3_4_7","1010.95" +"alexnet","64","0_1_2_3_4_7","5832.19" +"resnet50","64","0_1_2_3_4_7","1878.99" +"vgg16","64","0_1_2_3_4_7","891.55" +"inception3","64","0_1_2_3_4_7","1258.44" +"alexnet","128","0_1_2_3_4_7","10751.64" +"resnet50","128","0_1_2_3_4_7","2164.80" +"vgg16","128","0_1_2_3_4_7","1013.30" +"inception3","128","0_1_2_3_4_7","1375.49" +"alexnet","32","1_3_4_5_6_7","2831.22" +"resnet50","32","1_3_4_5_6_7","1434.37" +"vgg16","32","1_3_4_5_6_7","648.39" +"inception3","32","1_3_4_5_6_7","1002.29" +"alexnet","64","1_3_4_5_6_7","5525.78" +"resnet50","64","1_3_4_5_6_7","1852.38" +"vgg16","64","1_3_4_5_6_7","872.50" +"inception3","64","1_3_4_5_6_7","1230.71" +"alexnet","128","1_3_4_5_6_7","10312.83" +"resnet50","128","1_3_4_5_6_7","2138.01" +"vgg16","128","1_3_4_5_6_7","981.04" +"inception3","128","1_3_4_5_6_7","1351.22" +"alexnet","32","1_2_4_5_6_7","2810.31" +"resnet50","32","1_2_4_5_6_7","1463.05" +"vgg16","32","1_2_4_5_6_7","657.05" +"inception3","32","1_2_4_5_6_7","1029.66" +"alexnet","64","1_2_4_5_6_7","5538.91" +"resnet50","64","1_2_4_5_6_7","1884.19" +"vgg16","64","1_2_4_5_6_7","877.05" +"inception3","64","1_2_4_5_6_7","1201.52" +"alexnet","128","1_2_4_5_6_7","10321.55" +"resnet50","128","1_2_4_5_6_7","2108.87" +"vgg16","128","1_2_4_5_6_7","996.37" +"inception3","128","1_2_4_5_6_7","1319.26" +"alexnet","32","1_2_3_4_6_7","2026.41" +"resnet50","32","1_2_3_4_6_7","1471.85" +"vgg16","32","1_2_3_4_6_7","651.58" +"inception3","32","1_2_3_4_6_7","1004.32" +"alexnet","64","1_2_3_4_6_7","3984.76" +"resnet50","64","1_2_3_4_6_7","1851.77" +"vgg16","64","1_2_3_4_6_7","851.02" +"inception3","64","1_2_3_4_6_7","1205.94" +"alexnet","128","1_2_3_4_6_7","7679.94" +"resnet50","128","1_2_3_4_6_7","2161.22" +"vgg16","128","1_2_3_4_6_7","1007.43" +"inception3","128","1_2_3_4_6_7","1399.52" +"alexnet","32","0_1_2_3_5_6_7","2341.50" +"resnet50","32","0_1_2_3_5_6_7","1667.38" +"vgg16","32","0_1_2_3_5_6_7","662.62" +"inception3","32","0_1_2_3_5_6_7","1155.65" +"alexnet","64","0_1_2_3_5_6_7","4561.96" +"resnet50","64","0_1_2_3_5_6_7","2040.63" +"vgg16","64","0_1_2_3_5_6_7","905.40" +"inception3","64","0_1_2_3_5_6_7","1378.83" +"alexnet","128","0_1_2_3_5_6_7","8374.93" +"resnet50","128","0_1_2_3_5_6_7","2442.46" +"vgg16","128","0_1_2_3_5_6_7","1107.72" +"inception3","128","0_1_2_3_5_6_7","1593.34" +"alexnet","32","0_1_2_4_5_6_7","2167.61" +"resnet50","32","0_1_2_4_5_6_7","1638.73" +"vgg16","32","0_1_2_4_5_6_7","557.73" +"inception3","32","0_1_2_4_5_6_7","1185.76" +"alexnet","64","0_1_2_4_5_6_7","4301.64" +"resnet50","64","0_1_2_4_5_6_7","2092.82" +"vgg16","64","0_1_2_4_5_6_7","818.48" +"inception3","64","0_1_2_4_5_6_7","1414.97" +"alexnet","128","0_1_2_4_5_6_7","8082.85" +"resnet50","128","0_1_2_4_5_6_7","2406.13" +"vgg16","128","0_1_2_4_5_6_7","1016.84" +"inception3","128","0_1_2_4_5_6_7","1598.77" +"alexnet","32","0_1_2_3_4_6_7","1962.71" +"resnet50","32","0_1_2_3_4_6_7","1638.99" +"vgg16","32","0_1_2_3_4_6_7","557.16" +"inception3","32","0_1_2_3_4_6_7","1129.51" +"alexnet","64","0_1_2_3_4_6_7","3897.41" +"resnet50","64","0_1_2_3_4_6_7","2045.99" +"vgg16","64","0_1_2_3_4_6_7","824.24" +"inception3","64","0_1_2_3_4_6_7","1426.86" +"alexnet","128","0_1_2_3_4_6_7","7585.63" +"resnet50","128","0_1_2_3_4_6_7","2342.38" +"vgg16","128","0_1_2_3_4_6_7","1031.54" +"inception3","128","0_1_2_3_4_6_7","1584.15" +"alexnet","32","0_1_2_3_4_5_7","2538.91" +"resnet50","32","0_1_2_3_4_5_7","1644.99" +"vgg16","32","0_1_2_3_4_5_7","675.31" +"inception3","32","0_1_2_3_4_5_7","1208.17" +"alexnet","64","0_1_2_3_4_5_7","5151.24" +"resnet50","64","0_1_2_3_4_5_7","2120.26" +"vgg16","64","0_1_2_3_4_5_7","954.93" +"inception3","64","0_1_2_3_4_5_7","1447.62" +"alexnet","128","0_1_2_3_4_5_7","9321.12" +"resnet50","128","0_1_2_3_4_5_7","2346.78" +"vgg16","128","0_1_2_3_4_5_7","1127.31" +"inception3","128","0_1_2_3_4_5_7","1628.69" +"alexnet","32","0_1_3_4_5_6_7","1998.59" +"resnet50","32","0_1_3_4_5_6_7","1612.78" +"vgg16","32","0_1_3_4_5_6_7","552.43" +"inception3","32","0_1_3_4_5_6_7","1165.81" +"alexnet","64","0_1_3_4_5_6_7","3885.38" +"resnet50","64","0_1_3_4_5_6_7","2091.77" +"vgg16","64","0_1_3_4_5_6_7","800.68" +"inception3","64","0_1_3_4_5_6_7","1428.81" +"alexnet","128","0_1_3_4_5_6_7","7403.87" +"resnet50","128","0_1_3_4_5_6_7","2425.72" +"vgg16","128","0_1_3_4_5_6_7","974.97" +"inception3","128","0_1_3_4_5_6_7","1549.14" +"alexnet","32","0_1_2_3_4_5_6","2549.69" +"resnet50","32","0_1_2_3_4_5_6","1687.53" +"vgg16","32","0_1_2_3_4_5_6","685.23" +"inception3","32","0_1_2_3_4_5_6","1181.25" +"alexnet","64","0_1_2_3_4_5_6","4965.54" +"resnet50","64","0_1_2_3_4_5_6","2024.21" +"vgg16","64","0_1_2_3_4_5_6","953.19" +"inception3","64","0_1_2_3_4_5_6","1435.29" +"alexnet","128","0_1_2_3_4_5_6","9345.32" +"resnet50","128","0_1_2_3_4_5_6","2485.68" +"vgg16","128","0_1_2_3_4_5_6","1143.87" +"inception3","128","0_1_2_3_4_5_6","1579.46" +"alexnet","32","0_2_3_4_5_6_7","2140.24" +"resnet50","32","0_2_3_4_5_6_7","1650.53" +"vgg16","32","0_2_3_4_5_6_7","560.76" +"inception3","32","0_2_3_4_5_6_7","1181.57" +"alexnet","64","0_2_3_4_5_6_7","4353.79" +"resnet50","64","0_2_3_4_5_6_7","2038.03" +"vgg16","64","0_2_3_4_5_6_7","816.42" +"inception3","64","0_2_3_4_5_6_7","1415.18" +"alexnet","128","0_2_3_4_5_6_7","8112.01" +"resnet50","128","0_2_3_4_5_6_7","2385.85" +"vgg16","128","0_2_3_4_5_6_7","993.87" +"inception3","128","0_2_3_4_5_6_7","1596.65" +"alexnet","32","1_2_3_4_5_6_7","2163.43" +"resnet50","32","1_2_3_4_5_6_7","1631.97" +"vgg16","32","1_2_3_4_5_6_7","557.85" +"inception3","32","1_2_3_4_5_6_7","1179.02" +"alexnet","64","1_2_3_4_5_6_7","4259.44" +"resnet50","64","1_2_3_4_5_6_7","2097.62" +"vgg16","64","1_2_3_4_5_6_7","828.59" +"inception3","64","1_2_3_4_5_6_7","1423.61" +"alexnet","128","1_2_3_4_5_6_7","8003.65" +"resnet50","128","1_2_3_4_5_6_7","2406.21" +"vgg16","128","1_2_3_4_5_6_7","1021.62" +"inception3","128","1_2_3_4_5_6_7","1595.09"